{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "4c2a6fa7",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/site-packages/PyPDF2/__init__.py:21: DeprecationWarning: PyPDF2 is deprecated. Please move to the pypdf library instead.\n",
" warnings.warn(\n"
]
}
],
"source": [
"import os\n",
"\n",
"from dotenv import load_dotenv\n",
"\n",
"from evoagentx.agents.agent_manager import AgentManager\n",
"from evoagentx.benchmark import HotPotQA\n",
"from evoagentx.core.callbacks import suppress_logger_info\n",
"from evoagentx.core.logging import logger\n",
"from evoagentx.evaluators import Evaluator\n",
"from evoagentx.models import OpenAILLM, OpenAILLMConfig\n",
"from evoagentx.optimizers import TextGradOptimizer\n",
"from evoagentx.prompts import StringTemplate\n",
"from evoagentx.workflow import SequentialWorkFlowGraph\n",
"from dotenv import load_dotenv\n",
"\n",
"from evoagentx.agents.agent_manager import AgentManager\n",
"from evoagentx.benchmark import MBPP\n",
"from evoagentx.core.callbacks import suppress_logger_info\n",
"from evoagentx.core.logging import logger\n",
"from evoagentx.evaluators import Evaluator\n",
"from evoagentx.models import OpenAILLM, OpenAILLMConfig\n",
"from evoagentx.optimizers import TextGradOptimizer\n",
"from evoagentx.prompts import StringTemplate\n",
"from evoagentx.workflow import SequentialWorkFlowGraph\n",
"\n",
"from evoagentx.models import OpenAILLMConfig, OpenAILLM\n",
"from evoagentx.workflow import SEWWorkFlowGraph, STRUCTUREWorkFlowGraph\n",
"from evoagentx.agents import AgentManager\n",
"from evoagentx.benchmark import HumanEval,AFlowMBPP\n",
"from evoagentx.evaluators import Evaluator \n",
"from evoagentx.optimizers import SEWOptimizer, STRUCTUREOptimizer\n",
"from evoagentx.optimizers.structure_optimizer import STRUCTUREWorkFlowScheme\n",
"from evoagentx.core.callbacks import suppress_logger_info\n",
"\n",
"from evoagentx.models import OpenAILLMConfig, OpenAILLM,AzureOpenAIConfig,LiteLLMConfig,LiteLLM\n",
"from evoagentx.workflow import SEWWorkFlowGraph \n",
"from evoagentx.agents import AgentManager\n",
"from evoagentx.benchmark import MBPPPLUS, AFlowMBPPPLUS\n",
"from evoagentx.evaluators import Evaluator \n",
"from evoagentx.optimizers import SEWOptimizer \n",
"from evoagentx.core.callbacks import suppress_logger_info\n",
"from evoagentx.benchmark import HumanEvalPLUS\n",
"from evoagentx.benchmark import SciCode\n",
"from copy import deepcopy\n",
"from evoagentx.benchmark import PertQA\n",
"from copy import deepcopy\n",
"from evoagentx.benchmark import MolQA\n",
"import nest_asyncio\n",
"nest_asyncio.apply()\n",
"\n",
"class HotPotQASplits(HotPotQA):\n",
"\n",
" def _load_data(self):\n",
" # load the original test data \n",
" super()._load_data()\n",
" # split the data into train, dev and test\n",
" import numpy as np \n",
" np.random.seed(42)\n",
" permutation = np.random.permutation(len(self._dev_data))\n",
" full_test_data = self._dev_data \n",
" # randomly select 10 samples for train, 40 for dev, and 100 for test\n",
" self._train_data = [full_test_data[idx] for idx in permutation[:50]]\n",
" self._dev_data = [full_test_data[idx] for idx in permutation[:50]]\n",
" self._test_data = [full_test_data[idx] for idx in permutation[50:550]]\n",
" self._fulldata = full_test_data\n",
"\n",
"\n",
"def collate_func(example: dict) -> dict:\n",
" problem = \"Question: {}\\n\\nAnswer:\".format(example[\"question\"])\n",
" return {\"problem\": problem}\n",
"\n",
"\n",
"hotpotqa_graph_data = {\n",
" \"goal\": \"Answer the question based on the context. The answer should be a direct response to the question, without including explanations or reasoning.\",\n",
" \"tasks\": [\n",
" {\n",
" \"name\": \"answer_generate\",\n",
" \"description\": \"Answer the question based on the context.\",\n",
" \"inputs\": [\n",
" {\"name\": \"problem\", \"type\": \"str\", \"required\": True, \"description\": \"The problem to solve.\"}\n",
" ],\n",
" \"outputs\": [\n",
" {\"name\": \"answer\", \"type\": \"str\", \"required\": True, \"description\": \"The answer to the problem.\"}\n",
" ],\n",
" \"prompt_template\": StringTemplate(instruction=\"Think step by step to answer the question. You should explain your thinking process in the 'thought' field, and provide the final answer in the 'answer' field.\\nFormat your output in xml format, such as xxx and xxx.\"),\n",
" \"parse_mode\": \"xml\"\n",
" }\n",
" ] \n",
"}\n",
"os.environ[\"AZURE_OPENAI_DEPLOYMENT_NAME\"] = \"gpt-4o-mini\"\n",
"os.environ[\"AZURE_OPENAI_ENDPOINT\"] = \"https://tianyuliu-hua-raredisea-resource.cognitiveservices.azure.com/\"\n",
"os.environ[\"AZURE_OPENAI_KEY\"] = \"2pa9h2ZIN1lQepFWwYADlXIKIansa9KPhxMoumeGbRQ08f2uDTXiJQQJ99BKACHYHv6XJ3w3AAAAACOGsQIt\"\n",
"os.environ[\"AZURE_OPENAI_API_VERSION\"] = \"2025-01-01-preview\"\n",
"llm_config = LiteLLMConfig(model=\"azure/\" + os.getenv(\"AZURE_OPENAI_DEPLOYMENT_NAME\"), # Azure model format\n",
" azure_endpoint=os.getenv(\"AZURE_OPENAI_ENDPOINT\"),\n",
" azure_key=os.getenv(\"AZURE_OPENAI_KEY\"),\n",
" api_version=os.getenv(\"AZURE_OPENAI_API_VERSION\", \"2024-12-01-preview\"), top_p=0.85, temperature=0.2, frequency_penalty=0.0, presence_penalty=0.0)\n",
"\n",
"executor_llm = LiteLLM(config=llm_config)\n",
"optimizer_llm = LiteLLM(config=llm_config)\n",
"llm = executor_llm"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "ad0efa03",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"evoagentx.optimizers.sew_optimizer.SEWOptimizer"
]
},
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"SEWOptimizer "
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "ad4b2024",
"metadata": {},
"outputs": [],
"source": [
"# difficult easy "
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "c95059f0",
"metadata": {},
"outputs": [],
"source": [
"from evoagentx.benchmark import HotPotQA"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "84efabfa",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[32m2026-01-10 17:32:09.571\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.benchmark.hitqa\u001b[0m:\u001b[36m_load_data_from_file\u001b[0m:\u001b[36m51\u001b[0m - \u001b[1mloading HotPotQA data from /home/tl688/pitl688/selfevolve/EvoAgentX/examples/antibiotic_pred/ec_train.json ...\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:09.662\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.benchmark.hitqa\u001b[0m:\u001b[36m_load_data_from_file\u001b[0m:\u001b[36m51\u001b[0m - \u001b[1mloading HotPotQA data from /home/tl688/pitl688/selfevolve/EvoAgentX/examples/antibiotic_pred/ec_train.json ...\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:09.751\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.benchmark.hitqa\u001b[0m:\u001b[36m_load_data_from_file\u001b[0m:\u001b[36m51\u001b[0m - \u001b[1mloading HotPotQA data from /home/tl688/pitl688/selfevolve/EvoAgentX/examples/antibiotic_pred/ec_test.json ...\u001b[0m\n"
]
}
],
"source": [
"# llm_config = OpenAILLMConfig(model=\"gpt-4o-mini-2024-07-18\", openai_key=OPENAI_API_KEY, top_p=0.85, temperature=0.2, frequency_penalty=0.0, presence_penalty=0.0)\n",
"# llm = OpenAILLM(config=llm_config)\n",
"llm = executor_llm\n",
"\n",
"# obtain SEW workflow \n",
"sew_graph = SEWWorkFlowGraph.from_dict(hotpotqa_graph_data)\n",
"agent_manager = AgentManager()\n",
"agent_manager.add_agents_from_workflow(sew_graph, executor_llm.config)\n",
"\n",
"benchmark = MolQA()\n",
"\n",
"# obtain Evaluator\n",
"evaluator = Evaluator(llm=llm, agent_manager=agent_manager, collate_func=collate_func, num_workers=20, verbose=True)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "d2bba683",
"metadata": {},
"outputs": [],
"source": [
"# import json\n",
"# # with open(\"../../MaAS/maas/ext/maas/data/humaneval_train.jsonl\", 'w') as f:\n",
"# # json.dump(humaneval._dev_data, f, indent=2) # indent=4 makes the JSON output more readable\n",
"\n",
"\n",
"# # with open(\"../../MaAS/maas/ext/maas/data/humaneval_test.jsonl\", 'w') as f:\n",
"# # json.dump(humaneval._test_data, f, indent=2) # indent=4 makes the JSON output more readable\n",
"\n",
"# with open(\"../../MaAS/maas/ext/maas/data/humaneval_train.jsonl\", 'w') as f:\n",
"# for obj in humaneval._dev_data:\n",
"# json_line = json.dumps(obj)\n",
"# f.write(json_line + '\\n')\n",
" \n",
"# with open(\"../../MaAS/maas/ext/maas/data/humaneval_test.jsonl\", 'w') as f:\n",
"# for obj in humaneval._test_data:\n",
"# json_line = json.dumps(obj)\n",
"# f.write(json_line + '\\n')\n"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "8598151b",
"metadata": {
"scrolled": false
},
"outputs": [
{
"data": {
"text/plain": [
"1"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(sew_graph.to_dict()['nodes'])"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "b1f7fc18",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"0"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(sew_graph.edges)"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "33859fa8",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[]"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"sew_graph.edges"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "3c048529",
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"# obtain SEWOptimizer after having more roles, default\n",
"optimizer = SEWOptimizer(\n",
" graph=sew_graph, \n",
" evaluator=evaluator, \n",
" llm=llm, \n",
" max_steps=20,\n",
" eval_rounds=3, \n",
" repr_scheme=\"python\", \n",
" optimize_mode=\"all\", \n",
" order=\"zero-order\",\n",
" max_rounds=20,\n",
")\n",
"\n",
"# with suppress_logger_info():\n",
"# metrics = optimizer.evaluate(dataset=humaneval, eval_mode=\"test\")\n",
"# print(\"Evaluation metrics: \", metrics)\n"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "9cbdd001",
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
"np.random.seed(2024)\n",
"out = np.random.choice(benchmark._train_data, size=50, replace=False)\n",
"benchmark._train_data = out\n",
"benchmark._dev_data = out"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "8b05058e",
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[32m2026-01-10 17:32:09.823\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m678\u001b[0m - \u001b[1mOptimizing the SEWWorkFlowGraph workflow with python representation.\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:09.823\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m682\u001b[0m - \u001b[1mRun initial evaluation on the original workflow ...\u001b[0m\n",
"Evaluating workflow: 2%|▏ | 1/50 [00:02<01:41, 2.06s/it]Task exception was never retrieved\n",
"future: exception=RuntimeError('Event loop is closed')>\n",
"Traceback (most recent call last):\n",
" File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/asyncio/tasks.py\", line 277, in __step\n",
" result = coro.send(None)\n",
" ^^^^^^^^^^^^^^^\n",
" File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/site-packages/litellm/utils.py\", line 873, in _client_async_logging_helper\n",
" GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(\n",
" File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/site-packages/litellm/litellm_core_utils/logging_worker.py\", line 322, in ensure_initialized_and_enqueue\n",
" self.enqueue(async_coroutine)\n",
" File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/site-packages/litellm/litellm_core_utils/logging_worker.py\", line 131, in enqueue\n",
" self._queue.put_nowait(task)\n",
" File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/asyncio/queues.py\", line 147, in put_nowait\n",
" self._wakeup_next(self._getters)\n",
" File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/asyncio/queues.py\", line 63, in _wakeup_next\n",
" waiter.set_result(None)\n",
" File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/asyncio/futures.py\", line 263, in set_result\n",
" self.__schedule_callbacks()\n",
" File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/asyncio/futures.py\", line 173, in __schedule_callbacks\n",
" self._loop.call_soon(callback, self, context=ctx)\n",
" File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/asyncio/base_events.py\", line 762, in call_soon\n",
" self._check_closed()\n",
" File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/asyncio/base_events.py\", line 520, in _check_closed\n",
" raise RuntimeError('Event loop is closed')\n",
"RuntimeError: Event loop is closed\n",
"Evaluating workflow: 4%|▍ | 2/50 [00:02<00:45, 1.05it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 10%|█ | 5/50 [00:02<00:14, 3.16it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 22%|██▏ | 11/50 [00:02<00:04, 9.30it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 36%|███▌ | 18/50 [00:03<00:02, 14.59it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 42%|████▏ | 21/50 [00:04<00:05, 4.89it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 50%|█████ | 25/50 [00:05<00:04, 6.23it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 64%|██████▍ | 32/50 [00:05<00:01, 10.70it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 74%|███████▍ | 37/50 [00:05<00:00, 14.07it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 78%|███████▊ | 39/50 [00:06<00:01, 7.80it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 82%|████████▏ | 41/50 [00:06<00:01, 5.61it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 90%|█████████ | 45/50 [00:07<00:00, 6.56it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 94%|█████████▍| 47/50 [00:07<00:00, 7.36it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 98%|█████████▊| 49/50 [00:08<00:00, 6.57it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 100%|██████████| 50/50 [00:08<00:00, 6.14it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"\u001b[32m2026-01-10 17:32:18.025\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m685\u001b[0m - \u001b[1mInitial metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.56}\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[32m2026-01-10 17:32:18.657\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.014 | Total tokens: 44852 | Current cost: $0.000 | Current tokens: 104\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:19.282\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.014 | Total tokens: 44989 | Current cost: $0.000 | Current tokens: 137\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:19.284\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 0: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:21.018\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.014 | Total tokens: 45241 | Current cost: $0.000 | Current tokens: 252\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:21.820\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.014 | Total tokens: 45586 | Current cost: $0.000 | Current tokens: 345\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:21.821\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36mparse_workflow_python_repr\u001b[0m:\u001b[36m403\u001b[0m - \u001b[33m\u001b[1mFailed to parse workflow string: 'llm_config'. Return the original workflow.\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:21.821\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 1: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:22.665\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.014 | Total tokens: 45702 | Current cost: $0.000 | Current tokens: 116\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:23.315\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.014 | Total tokens: 45882 | Current cost: $0.000 | Current tokens: 180\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:23.317\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 2: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:23.929\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.014 | Total tokens: 45975 | Current cost: $0.000 | Current tokens: 93\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:24.587\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.014 | Total tokens: 46113 | Current cost: $0.000 | Current tokens: 138\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:24.588\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 3: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:25.185\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.014 | Total tokens: 46227 | Current cost: $0.000 | Current tokens: 114\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:25.782\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.014 | Total tokens: 46365 | Current cost: $0.000 | Current tokens: 138\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:25.783\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 4: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:26.398\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.014 | Total tokens: 46444 | Current cost: $0.000 | Current tokens: 79\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:27.046\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.014 | Total tokens: 46590 | Current cost: $0.000 | Current tokens: 146\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:27.047\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36mparse_workflow_python_repr\u001b[0m:\u001b[36m403\u001b[0m - \u001b[33m\u001b[1mFailed to parse workflow string: 'llm_config'. Return the original workflow.\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:27.047\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 5: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:28.261\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.014 | Total tokens: 46668 | Current cost: $0.000 | Current tokens: 78\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:28.900\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.014 | Total tokens: 46809 | Current cost: $0.000 | Current tokens: 141\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:28.901\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 6: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:29.546\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.014 | Total tokens: 46891 | Current cost: $0.000 | Current tokens: 82\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:30.199\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.014 | Total tokens: 47032 | Current cost: $0.000 | Current tokens: 141\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:30.200\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 7: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:30.906\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.014 | Total tokens: 47129 | Current cost: $0.000 | Current tokens: 97\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:31.945\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.014 | Total tokens: 47345 | Current cost: $0.000 | Current tokens: 216\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:31.946\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36mparse_workflow_python_repr\u001b[0m:\u001b[36m403\u001b[0m - \u001b[33m\u001b[1mFailed to parse workflow string: 'llm_config'. Return the original workflow.\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:31.946\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 8: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:32.769\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.015 | Total tokens: 47442 | Current cost: $0.000 | Current tokens: 97\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:33.417\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.015 | Total tokens: 47603 | Current cost: $0.000 | Current tokens: 161\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:33.418\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 9: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:34.022\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.015 | Total tokens: 47670 | Current cost: $0.000 | Current tokens: 67\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:34.719\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.015 | Total tokens: 47808 | Current cost: $0.000 | Current tokens: 138\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:34.720\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 10: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:35.311\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.015 | Total tokens: 47887 | Current cost: $0.000 | Current tokens: 79\u001b[0m\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[32m2026-01-10 17:32:36.582\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.015 | Total tokens: 48028 | Current cost: $0.000 | Current tokens: 141\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:36.584\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 11: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:38.113\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.015 | Total tokens: 48229 | Current cost: $0.000 | Current tokens: 201\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:38.758\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.015 | Total tokens: 48498 | Current cost: $0.000 | Current tokens: 269\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:38.759\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 12: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:39.357\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.015 | Total tokens: 48587 | Current cost: $0.000 | Current tokens: 89\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:40.103\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.015 | Total tokens: 48758 | Current cost: $0.000 | Current tokens: 171\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:40.104\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36mparse_workflow_python_repr\u001b[0m:\u001b[36m403\u001b[0m - \u001b[33m\u001b[1mFailed to parse workflow string: 'llm_config'. Return the original workflow.\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:40.105\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 13: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:40.917\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.015 | Total tokens: 48880 | Current cost: $0.000 | Current tokens: 122\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:41.664\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.015 | Total tokens: 49065 | Current cost: $0.000 | Current tokens: 185\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:41.665\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36mparse_workflow_python_repr\u001b[0m:\u001b[36m403\u001b[0m - \u001b[33m\u001b[1mFailed to parse workflow string: 'llm_config'. Return the original workflow.\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:41.665\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 14: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:43.439\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.015 | Total tokens: 49322 | Current cost: $0.000 | Current tokens: 257\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:43.892\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.015 | Total tokens: 49633 | Current cost: $0.000 | Current tokens: 311\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:43.894\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 15: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:44.466\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.015 | Total tokens: 49744 | Current cost: $0.000 | Current tokens: 111\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:45.222\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.015 | Total tokens: 49882 | Current cost: $0.000 | Current tokens: 138\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:45.223\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 16: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:45.655\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.015 | Total tokens: 49954 | Current cost: $0.000 | Current tokens: 72\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:46.290\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.015 | Total tokens: 50081 | Current cost: $0.000 | Current tokens: 127\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:46.292\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 17: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:46.742\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.015 | Total tokens: 50149 | Current cost: $0.000 | Current tokens: 68\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:47.433\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.015 | Total tokens: 50276 | Current cost: $0.000 | Current tokens: 127\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:47.434\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 18: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:48.020\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.015 | Total tokens: 50353 | Current cost: $0.000 | Current tokens: 77\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:48.616\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.015 | Total tokens: 50492 | Current cost: $0.000 | Current tokens: 139\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:48.617\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 19: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:48.617\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m707\u001b[0m - \u001b[1mReach the maximum number of steps 20. Stop the optimization.\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:48.618\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m710\u001b[0m - \u001b[1mRestore the best graph from the snapshot ...\u001b[0m\n",
"\u001b[32m2026-01-10 17:32:48.618\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36mrestore_best_graph\u001b[0m:\u001b[36m814\u001b[0m - \u001b[1mRestore the best graph from snapshot with metrics {'f1': 0.0, 'em': 0.0, 'acc': 0.56} ...\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 0%| | 1/3000 [00:01<1:27:45, 1.76s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 0%| | 5/3000 [00:02<15:39, 3.19it/s] "
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 0%| | 8/3000 [00:02<08:58, 5.55it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 0%| | 12/3000 [00:02<06:20, 7.85it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 0%| | 14/3000 [00:02<05:15, 9.46it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 1%| | 16/3000 [00:03<05:45, 8.65it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 1%| | 18/3000 [00:03<06:14, 7.96it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 1%| | 20/3000 [00:03<08:19, 5.97it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 1%| | 22/3000 [00:04<07:33, 6.56it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 1%| | 23/3000 [00:04<08:34, 5.79it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 1%| | 29/3000 [00:04<04:56, 10.02it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 1%| | 33/3000 [00:05<05:43, 8.65it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 1%| | 37/3000 [00:05<05:30, 8.97it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 1%|▏ | 39/3000 [00:06<06:24, 7.69it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 1%|▏ | 41/3000 [00:06<06:41, 7.38it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 1%|▏ | 43/3000 [00:06<07:30, 6.57it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 2%|▏ | 46/3000 [00:07<05:37, 8.75it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 2%|▏ | 48/3000 [00:07<05:07, 9.61it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 2%|▏ | 50/3000 [00:07<05:13, 9.41it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 2%|▏ | 53/3000 [00:07<04:33, 10.79it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 2%|▏ | 56/3000 [00:08<05:55, 8.27it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 2%|▏ | 58/3000 [00:08<09:01, 5.43it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 2%|▏ | 62/3000 [00:09<06:35, 7.44it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 2%|▏ | 64/3000 [00:09<05:53, 8.29it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 2%|▏ | 66/3000 [00:09<05:13, 9.35it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 2%|▏ | 70/3000 [00:10<04:28, 10.92it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 2%|▏ | 74/3000 [00:10<06:24, 7.60it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 3%|▎ | 79/3000 [00:11<04:47, 10.15it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 3%|▎ | 81/3000 [00:11<07:47, 6.25it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 3%|▎ | 83/3000 [00:12<07:02, 6.90it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 3%|▎ | 87/3000 [00:12<06:39, 7.29it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 3%|▎ | 91/3000 [00:12<04:51, 9.97it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 3%|▎ | 93/3000 [00:13<06:13, 7.78it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 3%|▎ | 95/3000 [00:13<06:54, 7.00it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 3%|▎ | 97/3000 [00:13<06:40, 7.24it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 3%|▎ | 99/3000 [00:14<07:18, 6.61it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 3%|▎ | 102/3000 [00:14<07:24, 6.53it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 4%|▎ | 105/3000 [00:14<05:02, 9.56it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 4%|▎ | 107/3000 [00:15<04:46, 10.09it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 4%|▎ | 111/3000 [00:15<06:13, 7.74it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 4%|▍ | 113/3000 [00:16<06:07, 7.85it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 4%|▍ | 115/3000 [00:16<06:06, 7.88it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 4%|▍ | 119/3000 [00:16<05:09, 9.31it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 4%|▍ | 121/3000 [00:16<05:13, 9.18it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 4%|▍ | 125/3000 [00:17<04:07, 11.61it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 4%|▍ | 127/3000 [00:17<06:35, 7.26it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 4%|▍ | 129/3000 [00:18<07:31, 6.36it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 4%|▍ | 130/3000 [00:18<07:55, 6.04it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 4%|▍ | 132/3000 [00:18<07:57, 6.00it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097694.871948225)])']\n",
"connector: \n",
"Evaluating workflow: 4%|▍ | 133/3000 [00:18<09:40, 4.94it/s]Unclosed connector\n",
"connections: ['deque([(, 2097695.159673983)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097694.905455873)])']\n",
"connector: \n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 5%|▍ | 142/3000 [00:19<03:58, 11.97it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 5%|▍ | 144/3000 [00:19<04:49, 9.88it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 5%|▍ | 146/3000 [00:20<05:45, 8.27it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 5%|▍ | 148/3000 [00:20<05:35, 8.50it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 5%|▌ | 151/3000 [00:20<06:06, 7.78it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 5%|▌ | 156/3000 [00:21<04:55, 9.61it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 5%|▌ | 158/3000 [00:21<06:15, 7.56it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 5%|▌ | 160/3000 [00:22<07:25, 6.38it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 5%|▌ | 164/3000 [00:22<05:26, 8.68it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 6%|▌ | 169/3000 [00:22<04:14, 11.10it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 6%|▌ | 172/3000 [00:22<04:09, 11.32it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 6%|▌ | 174/3000 [00:23<04:14, 11.12it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 6%|▌ | 176/3000 [00:23<06:46, 6.94it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 6%|▌ | 179/3000 [00:24<08:15, 5.69it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 6%|▌ | 184/3000 [00:24<04:28, 10.50it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 6%|▌ | 186/3000 [00:24<04:50, 9.69it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 6%|▋ | 188/3000 [00:25<06:49, 6.86it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 6%|▋ | 190/3000 [00:25<07:21, 6.36it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 6%|▋ | 194/3000 [00:26<05:59, 7.81it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 7%|▋ | 196/3000 [00:26<06:07, 7.62it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 7%|▋ | 197/3000 [00:26<09:01, 5.18it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 7%|▋ | 198/3000 [00:27<09:10, 5.09it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 7%|▋ | 203/3000 [00:27<04:59, 9.34it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 7%|▋ | 208/3000 [00:27<04:14, 10.99it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
" {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 7%|▋ | 211/3000 [00:28<06:11, 7.50it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 7%|▋ | 213/3000 [00:28<07:40, 6.05it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 7%|▋ | 215/3000 [00:29<06:30, 7.14it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 7%|▋ | 221/3000 [00:29<04:33, 10.17it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 7%|▋ | 223/3000 [00:29<05:08, 8.99it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 8%|▊ | 227/3000 [00:30<04:45, 9.70it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 8%|▊ | 231/3000 [00:30<05:09, 8.96it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 8%|▊ | 233/3000 [00:31<08:39, 5.32it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 8%|▊ | 238/3000 [00:31<05:58, 7.71it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 8%|▊ | 243/3000 [00:32<04:01, 11.44it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 8%|▊ | 248/3000 [00:32<04:35, 9.99it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 8%|▊ | 252/3000 [00:33<05:39, 8.09it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 9%|▊ | 257/3000 [00:34<05:55, 7.71it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 9%|▊ | 259/3000 [00:34<05:09, 8.86it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 9%|▊ | 262/3000 [00:34<05:32, 8.23it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 9%|▉ | 264/3000 [00:35<06:46, 6.73it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"\u001b[32m2026-01-10 17:33:27.830\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
"Example: {'COADD_ID': 'CO-ADD:0148671', 'INHIB_AVE': 11.18, 'SMILES': 'c1c(C(CC(=O)N2)c(snc3c4cccc(F)c4)c23)cc(c5c1)OCO5', 'HIT': 0.0, 'question': 'Is c1c(C(CC(=O)N2)c(snc3c4cccc(F)c4)c23)cc(c5c1)OCO5 capable of inhibiting the growth or survival of E. coli?', 'answer': 'No', '_id': 14135}\n",
"Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 4 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 9%|▉ | 266/3000 [00:39<27:55, 1.63it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[32m2026-01-10 17:33:31.060\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
"Example: {'COADD_ID': 'CO-ADD:0277736', 'INHIB_AVE': -5.91, 'SMILES': 'c12c(c3c(cccc3)OC1=O)nc(c(cc4)ccc4C(C)C)nc2NC(C)=O', 'HIT': 0.0, 'question': 'Can c12c(c3c(cccc3)OC1=O)nc(c(cc4)ccc4C(C)C)nc2NC(C)=O effectively kill or suppress Escherichia coli cells?', 'answer': 'No', '_id': 73552}\n",
"Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 9%|▉ | 267/3000 [00:42<46:44, 1.03s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[32m2026-01-10 17:33:31.283\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
"Example: {'COADD_ID': 'CO-ADD:0225640', 'INHIB_AVE': 2.95, 'SMILES': 'C1(=N2)N(C=CN=C1Oc(cc3)ccc3C)C(N2CC(=O)Nc4ccc(cc4F)Br)=O', 'HIT': 0.0, 'question': 'Does the proposed molecule C1(=N2)N(C=CN=C1Oc(cc3)ccc3C)C(N2CC(=O)Nc4ccc(cc4F)Br)=O exhibit antibacterial activity against Escherichia coli?', 'answer': 'No', '_id': 44869}\n",
"Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 3 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 9%|▉ | 268/3000 [00:42<39:58, 1.14it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[32m2026-01-10 17:33:31.400\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
"Example: {'COADD_ID': 'CO-ADD:0240505', 'INHIB_AVE': 22.96, 'SMILES': 'C(C#N)(C1c2scc(Br)c2)=C(N)N(C(CCCC3=O)=C13)c4ccccc4', 'HIT': 0.0, 'question': 'Is C(C#N)(C1c2scc(Br)c2)=C(N)N(C(CCCC3=O)=C13)c4ccccc4 capable of inhibiting the growth or survival of E. coli?', 'answer': 'No', '_id': 1627}\n",
"Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 3 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 9%|▉ | 269/3000 [00:42<32:45, 1.39it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[32m2026-01-10 17:33:31.418\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
"Example: {'COADD_ID': 'CO-ADD:0331221', 'INHIB_AVE': 0.72, 'SMILES': 'n1(CC=C)c(SCC(=O)Nc(cc2)ccc2C(=O)OC(C)C)nnc1c3ccco3', 'HIT': 0.0, 'question': 'Can exposure to n1(CC=C)c(SCC(=O)Nc(cc2)ccc2C(=O)OC(C)C)nnc1c3ccco3 lead to reduced viability of E. coli?', 'answer': 'No', '_id': 54634}\n",
"Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 3 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
"\u001b[32m2026-01-10 17:33:31.429\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
"Example: {'COADD_ID': 'CO-ADD:0217226', 'INHIB_AVE': 8.1, 'SMILES': 'C(F)(F)(COc1ccccc1NC(=O)c2ccc(cc2)C)C(F)F', 'HIT': 0.0, 'question': 'Can C(F)(F)(COc1ccccc1NC(=O)c2ccc(cc2)C)C(F)F effectively kill or suppress Escherichia coli cells?', 'answer': 'No', '_id': 22842}\n",
"Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 3 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
"\u001b[32m2026-01-10 17:33:31.484\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
"Example: {'COADD_ID': 'CO-ADD:0234228', 'INHIB_AVE': 10.17, 'SMILES': 'c1(c(O)c(\\\\C=N\\\\c2ccc(cc2Cl)Cl)c(c(Cl)c1C)C)[N+](=O)[O-]', 'HIT': 0.0, 'question': 'Does c1(c(O)c(\\\\C=N\\\\c2ccc(cc2Cl)Cl)c(c(Cl)c1C)C)[N+](=O)[O-] show antimicrobial efficacy specifically against Escherichia coli strains?', 'answer': 'No', '_id': 16564}\n",
"Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 4 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
"\u001b[32m2026-01-10 17:33:31.643\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
"Example: {'COADD_ID': 'CO-ADD:0326254', 'INHIB_AVE': 5.32, 'SMILES': 'N1(c2ccc(cc2)Cl)C(=O)CC(N(C3CCCC3)C(C)=O)C1=O', 'HIT': 0.0, 'question': 'Is N1(c2ccc(cc2)Cl)C(=O)CC(N(C3CCCC3)C(C)=O)C1=O effective in eliminating E. coli under experimental conditions?', 'answer': 'No', '_id': 34235}\n",
"Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 4 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 9%|▉ | 273/3000 [00:42<16:26, 2.76it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[32m2026-01-10 17:33:31.683\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
"Example: {'COADD_ID': 'CO-ADD:0161257', 'INHIB_AVE': -7.52, 'SMILES': 'C1(=O)N(\\\\N=C\\\\c(cccc2[N+](=O)[O-])c2)C(C)=CC(C)=C1C(N)=O', 'HIT': 0.0, 'question': 'Is C1(=O)N(\\\\N=C\\\\c(cccc2[N+](=O)[O-])c2)C(C)=CC(C)=C1C(N)=O capable of inhibiting the growth or survival of E. coli?', 'answer': 'No', '_id': 76097}\n",
"Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 4 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
"\u001b[32m2026-01-10 17:33:31.771\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
"Example: {'COADD_ID': 'CO-ADD:0303166', 'INHIB_AVE': 0.06, 'SMILES': 'N1(c2ccc(cc2)OC)C(=O)NC(\\\\C(=C/c3cc(Br)ccc3OCCOc(cc(CC)cc4C)c4)\\\\C1=O)=O', 'HIT': 0.0, 'question': 'Does N1(c2ccc(cc2)OC)C(=O)NC(\\\\C(=C/c3cc(Br)ccc3OCCOc(cc(CC)cc4C)c4)\\\\C1=O)=O show antimicrobial efficacy specifically against Escherichia coli strains?', 'answer': 'No', '_id': 57221}\n",
"Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 4 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 9%|▉ | 275/3000 [00:43<12:38, 3.59it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[32m2026-01-10 17:33:31.781\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
"Example: {'COADD_ID': 'CO-ADD:0277496', 'INHIB_AVE': -1.48, 'SMILES': 'C1(C(OCCOC)=O)=C(C)NC(NC1c(cc2)cc(c2OC(F)F)OC)=O', 'HIT': 0.0, 'question': 'Can exposure to C1(C(OCCOC)=O)=C(C)NC(NC1c(cc2)cc(c2OC(F)F)OC)=O lead to reduced viability of E. coli?', 'answer': 'No', '_id': 62675}\n",
"Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 4 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
"\u001b[32m2026-01-10 17:33:31.929\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
"Example: {'COADD_ID': 'CO-ADD:0162954', 'INHIB_AVE': -7.52, 'SMILES': 'S(=O)(=O)(CCC(C)=O)c1ccc(cc1)Cc(cc2)ccc2S(=O)(=O)CCC(C)=O', 'HIT': 0.0, 'question': 'Is Escherichia coli susceptible to treatment with the compound S(=O)(=O)(CCC(C)=O)c1ccc(cc1)Cc(cc2)ccc2S(=O)(=O)CCC(C)=O?', 'answer': 'No', '_id': 76090}\n",
"Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 4 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 9%|▉ | 277/3000 [00:43<10:03, 4.51it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[32m2026-01-10 17:33:31.980\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
"Example: {'COADD_ID': 'CO-ADD:0222713', 'INHIB_AVE': 16.99, 'SMILES': 'N1(CNc2ccccc2C(OC)=O)C(=O)\\\\C(\\\\SC1=O)=C/c(cc3OC)cc(c3OC)OC', 'HIT': 0.0, 'question': 'Does N1(CNc2ccccc2C(OC)=O)C(=O)\\\\C(\\\\SC1=O)=C/c(cc3OC)cc(c3OC)OC show antimicrobial efficacy specifically against Escherichia coli strains?', 'answer': 'No', '_id': 5300}\n",
"Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 4 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
"\u001b[32m2026-01-10 17:33:32.084\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
"Example: {'COADD_ID': 'CO-ADD:0326010', 'INHIB_AVE': -1.13, 'SMILES': 'S(=O)(=O)(Nc(cc1)ccc1OCC)c2ccc(c(C)c2Cl)Cl', 'HIT': 0.0, 'question': 'Is Escherichia coli susceptible to treatment with the compound S(=O)(=O)(Nc(cc1)ccc1OCC)c2ccc(c(C)c2Cl)Cl?', 'answer': 'No', '_id': 61539}\n",
"Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 3 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 9%|▉ | 279/3000 [00:43<08:09, 5.56it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[32m2026-01-10 17:33:32.149\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
"Example: {'COADD_ID': 'CO-ADD:0156154', 'INHIB_AVE': 11.42, 'SMILES': 'O=C(C(C=C1)=CC=C1Cl)N2C3=C(SC4=C2C=CC=C4)C=CC(Cl)=C3', 'HIT': 0.0, 'question': 'Is O=C(C(C=C1)=CC=C1Cl)N2C3=C(SC4=C2C=CC=C4)C=CC(Cl)=C3 capable of inhibiting the growth or survival of E. coli?', 'answer': 'No', '_id': 13619}\n",
"Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 4 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
"\u001b[32m2026-01-10 17:33:33.003\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
"Example: {'COADD_ID': 'CO-ADD:0236947', 'INHIB_AVE': 3.12, 'SMILES': 'O=C(C1=CC=C(OC)C=C1CC/2)C2=CC3=CC=CN=C3Br', 'HIT': 0.0, 'question': 'Is O=C(C1=CC=C(OC)C=C1CC/2)C2=CC3=CC=CN=C3Br capable of inhibiting the growth or survival of E. coli?', 'answer': 'No', '_id': 44066}\n",
"Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 7 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 9%|▉ | 281/3000 [00:44<11:51, 3.82it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[32m2026-01-10 17:33:33.069\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
"Example: {'COADD_ID': 'CO-ADD:0233044', 'INHIB_AVE': -1.93, 'SMILES': 'N1(c2ccccc2c3ccccc3)C(=O)c(c4C1=O)ccc(c4)C(=O)Nc5ccc(cc5C(O)=O)Br', 'HIT': 0.0, 'question': 'Is Escherichia coli susceptible to treatment with the compound N1(c2ccccc2c3ccccc3)C(=O)c(c4C1=O)ccc(c4)C(=O)Nc5ccc(cc5C(O)=O)Br?', 'answer': 'No', '_id': 64166}\n",
"Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 7 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
"\u001b[32m2026-01-10 17:33:33.180\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
"Example: {'COADD_ID': 'CO-ADD:0287710', 'INHIB_AVE': -0.2, 'SMILES': 'N1(\\\\N=C(/CCCCC2)\\\\C2)C(=O)c3c(NC1c4ccccc4Cl)cccc3', 'HIT': 0.0, 'question': 'Does treatment with N1(\\\\N=C(/CCCCC2)\\\\C2)C(=O)c3c(NC1c4ccccc4Cl)cccc3 result in bacteriostatic or bactericidal effects on E. coli?', 'answer': 'No', '_id': 58255}\n",
"Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 7 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 9%|▉ | 283/3000 [00:44<09:32, 4.75it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[32m2026-01-10 17:33:33.348\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
"Example: {'COADD_ID': 'CO-ADD:0166323', 'INHIB_AVE': 14.81, 'SMILES': 'C(OC)(COc(cc1)ccc1C=O)=O', 'HIT': 0.0, 'question': 'Is Escherichia coli susceptible to treatment with the compound C(OC)(COc(cc1)ccc1C=O)=O?', 'answer': 'No', '_id': 7841}\n",
"Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 7 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 9%|▉ | 284/3000 [00:44<09:12, 4.92it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[32m2026-01-10 17:33:35.088\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
"Example: {'COADD_ID': 'CO-ADD:0296119', 'INHIB_AVE': 0.49, 'SMILES': 'c1(cc(F)c(cc1Cl)F)C(=O)N(CC2)CCN2c(cc3)c4c(c3[N+](=O)[O-])cccn4', 'HIT': 0.0, 'question': 'Does c1(cc(F)c(cc1Cl)F)C(=O)N(CC2)CCN2c(cc3)c4c(c3[N+](=O)[O-])cccn4 demonstrate bactericidal effects against E. coli in vitro?', 'answer': 'No', '_id': 55543}\n",
"Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 5 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 10%|▉ | 285/3000 [00:46<23:02, 1.96it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[32m2026-01-10 17:33:35.503\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
"Example: {'COADD_ID': 'CO-ADD:0161314', 'INHIB_AVE': 2.25, 'SMILES': 'C(C#N)(C1c(ccc(c2[N+](=O)[O-])Cl)c2)=C(N)SC(N)=C1C#N', 'HIT': 0.0, 'question': 'Can C(C#N)(C1c(ccc(c2[N+](=O)[O-])Cl)c2)=C(N)SC(N)=C1C#N effectively kill or suppress Escherichia coli cells?', 'answer': 'No', '_id': 47986}\n",
"Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 4 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 10%|▉ | 286/3000 [00:46<22:05, 2.05it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[32m2026-01-10 17:33:38.281\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 10%|▉ | 287/3000 [00:49<46:57, 1.04s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"\u001b[32m2026-01-10 17:33:38.535\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 10%|▉ | 288/3000 [00:49<37:52, 1.19it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"\u001b[32m2026-01-10 17:33:38.555\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
"\u001b[32m2026-01-10 17:33:38.560\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"\u001b[32m2026-01-10 17:33:38.621\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"\u001b[32m2026-01-10 17:33:38.779\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 10%|▉ | 292/3000 [00:50<17:00, 2.65it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"\u001b[32m2026-01-10 17:33:39.060\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 10%|▉ | 293/3000 [00:50<16:15, 2.78it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"\u001b[32m2026-01-10 17:33:39.113\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"\u001b[32m2026-01-10 17:33:39.225\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 10%|▉ | 295/3000 [00:50<12:04, 3.74it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"\u001b[32m2026-01-10 17:33:39.319\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"\u001b[32m2026-01-10 17:33:39.479\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 10%|▉ | 297/3000 [00:50<10:00, 4.50it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"\u001b[32m2026-01-10 17:33:39.644\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 10%|▉ | 298/3000 [00:50<09:31, 4.73it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"\u001b[32m2026-01-10 17:33:40.417\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 4 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 10%|▉ | 299/3000 [00:51<14:51, 3.03it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"\u001b[32m2026-01-10 17:33:40.665\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 3 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 10%|█ | 300/3000 [00:51<13:59, 3.22it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"\u001b[32m2026-01-10 17:33:40.698\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 3 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"\u001b[32m2026-01-10 17:33:40.702\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 3 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"\u001b[32m2026-01-10 17:33:40.751\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 3 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"\u001b[32m2026-01-10 17:33:41.138\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 3 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 10%|█ | 304/3000 [00:52<09:02, 4.97it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"\u001b[32m2026-01-10 17:33:41.884\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
"Example: {'COADD_ID': 'CO-ADD:0233758', 'INHIB_AVE': 8.35, 'SMILES': 'N1(CCC[C@@H](C1)c(ccn2)o2)C(=O)[C@@H]3CCCN3.Cl', 'HIT': 0.0, 'question': 'Can N1(CCC[C@@H](C1)c(ccn2)o2)C(=O)[C@@H]3CCCN3.Cl effectively kill or suppress Escherichia coli cells?', 'answer': 'No', '_id': 21942}\n",
"Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 4 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 10%|█ | 305/3000 [00:53<13:10, 3.41it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[32m2026-01-10 17:33:41.959\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
"Example: {'COADD_ID': 'CO-ADD:0256464', 'INHIB_AVE': -9.4, 'SMILES': 'c1(cnc(C)n1CC(NN)=O)[N+](=O)[O-]', 'HIT': 0.0, 'question': 'Can c1(cnc(C)n1CC(NN)=O)[N+](=O)[O-] act as an antibacterial agent targeting Escherichia coli?', 'answer': 'No', '_id': 78235}\n",
"Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 4 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
"\u001b[32m2026-01-10 17:33:42.107\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
"Example: {'COADD_ID': 'CO-ADD:0215412', 'INHIB_AVE': 2.96, 'SMILES': 'c(C#N)(c1c2ccccc2)c(SCc(ccc(c3Cl)Cl)c3)nc(N)c1C#N', 'HIT': 0.0, 'question': 'Does c(C#N)(c1c2ccccc2)c(SCc(ccc(c3Cl)Cl)c3)nc(N)c1C#N show antimicrobial efficacy specifically against Escherichia coli strains?', 'answer': 'No', '_id': 44825}\n",
"Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 4 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 10%|█ | 307/3000 [00:53<10:30, 4.27it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[32m2026-01-10 17:33:42.247\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
"Example: {'COADD_ID': 'CO-ADD:0232115', 'INHIB_AVE': 5.87, 'SMILES': 'C(=O)(c1ccc(cc1)Cl)NCC(=O)N\\\\N=C\\\\c(ccc(c23)OCO2)c3', 'HIT': 0.0, 'question': 'Is Escherichia coli susceptible to treatment with the compound C(=O)(c1ccc(cc1)Cl)NCC(=O)N\\\\N=C\\\\c(ccc(c23)OCO2)c3?', 'answer': 'No', '_id': 31806}\n",
"Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 3 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 10%|█ | 308/3000 [00:53<09:42, 4.62it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[32m2026-01-10 17:33:43.613\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 10%|█ | 309/3000 [00:54<20:39, 2.17it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"\u001b[32m2026-01-10 17:33:43.774\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 10%|█ | 310/3000 [00:55<17:31, 2.56it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"\u001b[32m2026-01-10 17:33:44.362\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 10%|█ | 311/3000 [00:55<19:44, 2.27it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"\u001b[32m2026-01-10 17:33:44.450\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"\u001b[32m2026-01-10 17:33:44.842\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 10%|█ | 313/3000 [00:56<15:59, 2.80it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"\u001b[32m2026-01-10 17:33:44.849\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"\u001b[32m2026-01-10 17:33:44.880\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"\u001b[32m2026-01-10 17:33:44.892\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"\u001b[32m2026-01-10 17:33:45.022\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 11%|█ | 317/3000 [00:56<08:23, 5.33it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"\u001b[32m2026-01-10 17:33:45.122\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"\u001b[32m2026-01-10 17:33:45.248\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 11%|█ | 319/3000 [00:56<07:26, 6.01it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"\u001b[32m2026-01-10 17:33:45.271\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"\u001b[32m2026-01-10 17:33:45.382\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
"\u001b[32m2026-01-10 17:33:45.383\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
"Example: {'COADD_ID': 'CO-ADD:0278342', 'INHIB_AVE': 0.69, 'SMILES': 'N1(c2cccc(Cl)c2Cl)C(=O)\\\\C(=C\\\\c3ccc(C)o3)\\\\C(NC1=S)=O', 'HIT': 0.0, 'question': 'Does treatment with N1(c2cccc(Cl)c2Cl)C(=O)\\\\C(=C\\\\c3ccc(C)o3)\\\\C(NC1=S)=O result in bacteriostatic or bactericidal effects on E. coli?', 'answer': 'No', '_id': 54760}\n",
"Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 3 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 11%|█ | 321/3000 [00:56<06:09, 7.24it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[32m2026-01-10 17:33:46.501\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 11%|█ | 323/3000 [00:57<11:36, 3.84it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"\u001b[32m2026-01-10 17:33:46.638\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 11%|█ | 324/3000 [00:57<10:40, 4.18it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"\u001b[32m2026-01-10 17:33:47.021\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 11%|█ | 325/3000 [00:58<11:55, 3.74it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"\u001b[32m2026-01-10 17:33:47.153\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 11%|█ | 326/3000 [00:58<10:35, 4.21it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"\u001b[32m2026-01-10 17:33:47.259\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 11%|█ | 327/3000 [00:58<09:11, 4.85it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"\u001b[32m2026-01-10 17:33:47.382\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 11%|█ | 328/3000 [00:58<08:13, 5.42it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"\u001b[32m2026-01-10 17:33:47.401\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"\u001b[32m2026-01-10 17:33:47.515\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 11%|█ | 330/3000 [00:58<06:00, 7.41it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"\u001b[32m2026-01-10 17:33:47.522\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"\u001b[32m2026-01-10 17:33:47.622\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
"Example: {'COADD_ID': 'CO-ADD:0155675', 'INHIB_AVE': -3.55, 'SMILES': '[O-][N+](C1=CC=CC=C1OCC(N/N=C/C2=CC=C(C3=CC=CC([N+]([O-])=O)=C3)O2)=O)=O', 'HIT': 0.0, 'question': 'Does [O-][N+](C1=CC=CC=C1OCC(N/N=C/C2=CC=C(C3=CC=CC([N+]([O-])=O)=C3)O2)=O)=O show antimicrobial efficacy specifically against Escherichia coli strains?', 'answer': 'No', '_id': 68660}\n",
"Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 11%|█ | 332/3000 [00:58<04:38, 9.57it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[32m2026-01-10 17:33:47.838\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
"Example: {'COADD_ID': 'CO-ADD:0232143', 'INHIB_AVE': 5.88, 'SMILES': 'C(=O)(NCC(=O)N\\\\N=C(/C)\\\\c1ccc(cc1)C)c2ccco2', 'HIT': 0.0, 'question': 'Is C(=O)(NCC(=O)N\\\\N=C(/C)\\\\c1ccc(cc1)C)c2ccco2 effective in eliminating E. coli under experimental conditions?', 'answer': 'No', '_id': 31731}\n",
"Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 11%|█ | 336/3000 [01:00<12:10, 3.65it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"\u001b[32m2026-01-10 17:33:49.636\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 11%|█▏ | 338/3000 [01:00<09:08, 4.85it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"\u001b[32m2026-01-10 17:33:49.765\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 11%|█▏ | 340/3000 [01:01<07:09, 6.19it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 11%|█▏ | 342/3000 [01:02<13:09, 3.37it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 11%|█▏ | 343/3000 [01:02<12:51, 3.45it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 12%|█▏ | 347/3000 [01:03<09:11, 4.81it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 12%|█▏ | 352/3000 [01:04<08:07, 5.44it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 12%|█▏ | 356/3000 [01:04<05:32, 7.96it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 12%|█▏ | 358/3000 [01:04<05:36, 7.84it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 12%|█▏ | 362/3000 [01:04<05:02, 8.73it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 12%|█▏ | 364/3000 [01:05<04:29, 9.77it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 12%|█▏ | 368/3000 [01:05<06:40, 6.58it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 12%|█▏ | 370/3000 [01:06<06:23, 6.86it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 12%|█▏ | 372/3000 [01:06<07:17, 6.00it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 12%|█▎ | 375/3000 [01:06<05:43, 7.64it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 13%|█▎ | 378/3000 [01:07<05:07, 8.53it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 13%|█▎ | 380/3000 [01:07<05:22, 8.12it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 13%|█▎ | 384/3000 [01:07<04:59, 8.75it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 13%|█▎ | 385/3000 [01:08<05:15, 8.29it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 13%|█▎ | 388/3000 [01:08<05:40, 7.67it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 13%|█▎ | 390/3000 [01:08<05:46, 7.54it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 13%|█▎ | 393/3000 [01:09<06:11, 7.02it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 13%|█▎ | 398/3000 [01:09<03:26, 12.58it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 13%|█▎ | 400/3000 [01:09<03:58, 10.91it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 13%|█▎ | 402/3000 [01:09<04:20, 9.96it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 14%|█▎ | 405/3000 [01:10<08:04, 5.35it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 14%|█▎ | 411/3000 [01:11<03:49, 11.28it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 14%|█▍ | 413/3000 [01:11<03:57, 10.90it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 14%|█▍ | 415/3000 [01:11<05:19, 8.09it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 14%|█▍ | 418/3000 [01:11<04:02, 10.64it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 14%|█▍ | 420/3000 [01:12<03:42, 11.59it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 14%|█▍ | 422/3000 [01:12<06:32, 6.56it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 14%|█▍ | 424/3000 [01:13<07:39, 5.61it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 14%|█▍ | 428/3000 [01:13<05:53, 7.27it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 14%|█▍ | 433/3000 [01:13<03:55, 10.92it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 15%|█▍ | 436/3000 [01:14<04:09, 10.29it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 15%|█▍ | 438/3000 [01:14<06:03, 7.05it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 15%|█▍ | 440/3000 [01:15<06:57, 6.13it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 15%|█▍ | 442/3000 [01:15<06:15, 6.80it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 15%|█▍ | 443/3000 [01:15<06:06, 6.97it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 15%|█▍ | 447/3000 [01:15<05:05, 8.35it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 15%|█▍ | 449/3000 [01:16<04:39, 9.13it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 15%|█▌ | 451/3000 [01:16<04:32, 9.37it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 15%|█▌ | 453/3000 [01:16<05:11, 8.18it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 15%|█▌ | 456/3000 [01:17<06:52, 6.16it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 15%|█▌ | 461/3000 [01:17<05:28, 7.73it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 15%|█▌ | 464/3000 [01:18<04:27, 9.46it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 16%|█▌ | 467/3000 [01:18<03:47, 11.12it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 16%|█▌ | 469/3000 [01:18<04:01, 10.48it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 16%|█▌ | 472/3000 [01:18<04:45, 8.86it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 16%|█▌ | 474/3000 [01:19<05:51, 7.18it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 16%|█▌ | 477/3000 [01:19<05:34, 7.55it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 16%|█▌ | 479/3000 [01:19<04:35, 9.15it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 16%|█▌ | 482/3000 [01:20<07:11, 5.84it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 16%|█▌ | 484/3000 [01:20<05:22, 7.80it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 16%|█▋ | 488/3000 [01:21<04:02, 10.35it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 16%|█▋ | 490/3000 [01:21<04:06, 10.17it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 16%|█▋ | 492/3000 [01:21<06:27, 6.47it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 16%|█▋ | 493/3000 [01:22<10:12, 4.09it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 17%|█▋ | 497/3000 [01:23<06:55, 6.02it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 17%|█▋ | 500/3000 [01:23<06:38, 6.28it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 17%|█▋ | 506/3000 [01:23<03:03, 13.57it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 17%|█▋ | 509/3000 [01:24<03:51, 10.77it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 17%|█▋ | 511/3000 [01:24<03:36, 11.51it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 17%|█▋ | 513/3000 [01:24<06:15, 6.63it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 17%|█▋ | 517/3000 [01:25<06:12, 6.67it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 17%|█▋ | 520/3000 [01:26<06:31, 6.33it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 17%|█▋ | 524/3000 [01:26<04:37, 8.92it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 18%|█▊ | 526/3000 [01:26<04:00, 10.27it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 18%|█▊ | 528/3000 [01:26<04:10, 9.86it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 18%|█▊ | 532/3000 [01:27<05:39, 7.26it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 18%|█▊ | 534/3000 [01:27<05:28, 7.51it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 18%|█▊ | 539/3000 [01:28<04:53, 8.37it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 18%|█▊ | 544/3000 [01:28<04:13, 9.69it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 18%|█▊ | 546/3000 [01:29<04:23, 9.31it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 18%|█▊ | 548/3000 [01:29<04:46, 8.56it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 18%|█▊ | 550/3000 [01:29<04:41, 8.69it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 18%|█▊ | 552/3000 [01:30<06:13, 6.55it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 18%|█▊ | 553/3000 [01:30<07:26, 5.48it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 18%|█▊ | 555/3000 [01:30<06:47, 6.01it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 19%|█▊ | 557/3000 [01:31<07:04, 5.76it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 19%|█▊ | 561/3000 [01:31<04:28, 9.08it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 19%|█▉ | 563/3000 [01:31<04:17, 9.46it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 19%|█▉ | 568/3000 [01:31<03:27, 11.70it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 19%|█▉ | 570/3000 [01:32<05:51, 6.91it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 19%|█▉ | 573/3000 [01:33<07:12, 5.61it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 19%|█▉ | 578/3000 [01:33<04:02, 9.99it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 19%|█▉ | 583/3000 [01:33<03:36, 11.15it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 20%|█▉ | 586/3000 [01:33<02:52, 14.00it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 20%|█▉ | 588/3000 [01:34<04:58, 8.09it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 20%|█▉ | 593/3000 [01:35<05:53, 6.82it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 20%|█▉ | 595/3000 [01:35<05:44, 6.99it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 20%|█▉ | 598/3000 [01:36<05:27, 7.34it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Evaluating workflow: 20%|██ | 601/3000 [01:36<05:35, 7.16it/s]Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097695.199769279)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097695.473596559)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097695.352257943)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097695.751721669)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097695.058871246)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097694.457078643)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097695.391940769)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097695.076186155)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097695.091972919)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097695.036603625)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097695.030669327)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097695.43318479)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097695.096205965)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097695.445775692)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097695.282213716)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097695.839455878)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097697.484573281)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097698.006550075)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097697.064096366)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097697.508680813)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097699.652659826)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097699.895076367)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097699.800058514)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097699.825371565)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097700.288237764)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097700.533241261)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097699.277893031)])']\n",
"connector: \n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Unclosed connector\n",
"connections: ['deque([(, 2097700.398903814)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097700.013124506)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097699.707131608)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097733.945262679)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097733.490579246)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097733.080230055)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097734.642944896)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097733.276366821)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097734.346525543)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097733.979476669)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097735.404317318)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097733.304579214)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097733.954019606)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097734.071394677)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097734.411640277)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097734.327106703)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097733.507360054)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097733.615553136)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097733.808525755)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097736.002233049)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097736.116433316)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097736.08346601)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097736.605640692)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097735.707893423)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097735.928419455)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097736.003416848)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097738.121596471)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097738.251584906)])']\n",
"connector: \n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097738.54080827)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097741.147427051)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097737.717460166)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097739.167082527)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097740.591460884)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097740.929533925)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097744.054695319)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097743.305958045)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097746.138833164)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097743.784974065)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097746.303999855)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097746.364009174)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097747.750131061)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097747.356587002)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097746.064856508)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097745.846524025)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097748.369081888)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097748.255387545)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097750.27266823)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097750.178740437)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097748.208971438)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097748.106459671)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097748.001517441)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097750.181111645)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 2097750.503915919)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: