{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2a8e89d4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/site-packages/PyPDF2/__init__.py:21: DeprecationWarning: PyPDF2 is deprecated. Please move to the pypdf library instead.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "from evoagentx.agents.agent_manager import AgentManager\n",
    "from evoagentx.benchmark import HotPotQA\n",
    "from evoagentx.core.callbacks import suppress_logger_info\n",
    "from evoagentx.core.logging import logger\n",
    "from evoagentx.evaluators import Evaluator\n",
    "from evoagentx.models import OpenAILLM, OpenAILLMConfig\n",
    "from evoagentx.optimizers import TextGradOptimizer\n",
    "from evoagentx.prompts import StringTemplate\n",
    "from evoagentx.workflow import SequentialWorkFlowGraph\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "from evoagentx.agents.agent_manager import AgentManager\n",
    "from evoagentx.benchmark import MBPP\n",
    "from evoagentx.core.callbacks import suppress_logger_info\n",
    "from evoagentx.core.logging import logger\n",
    "from evoagentx.evaluators import Evaluator\n",
    "from evoagentx.models import OpenAILLM, OpenAILLMConfig\n",
    "from evoagentx.optimizers import TextGradOptimizer\n",
    "from evoagentx.prompts import StringTemplate\n",
    "from evoagentx.workflow import SequentialWorkFlowGraph\n",
    "\n",
    "from evoagentx.models import OpenAILLMConfig, OpenAILLM\n",
    "from evoagentx.workflow import SEWWorkFlowGraph, STRUCTUREWorkFlowGraph\n",
    "from evoagentx.agents import AgentManager\n",
    "from evoagentx.benchmark import HumanEval,AFlowMBPP\n",
    "from evoagentx.evaluators import Evaluator \n",
    "from evoagentx.optimizers import SEWOptimizer, STRUCTUREOptimizer\n",
    "from evoagentx.optimizers.structure_optimizer import STRUCTUREWorkFlowScheme\n",
    "from evoagentx.core.callbacks import suppress_logger_info\n",
    "\n",
    "from evoagentx.models import OpenAILLMConfig, OpenAILLM,AzureOpenAIConfig,LiteLLMConfig,LiteLLM\n",
    "from evoagentx.workflow import SEWWorkFlowGraph \n",
    "from evoagentx.agents import AgentManager\n",
    "from evoagentx.benchmark import MBPPPLUS, AFlowMBPPPLUS\n",
    "from evoagentx.evaluators import Evaluator \n",
    "from evoagentx.optimizers import SEWOptimizer \n",
    "from evoagentx.core.callbacks import suppress_logger_info\n",
    "from evoagentx.benchmark import HumanEvalPLUS\n",
    "from evoagentx.benchmark import SciCode\n",
    "from copy import deepcopy\n",
    "\n",
    "import nest_asyncio\n",
    "nest_asyncio.apply()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "54f40417",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "from evoagentx.agents.agent_manager import AgentManager\n",
    "from evoagentx.benchmark import HotPotQA\n",
    "from evoagentx.core.callbacks import suppress_logger_info\n",
    "from evoagentx.core.logging import logger\n",
    "from evoagentx.evaluators import Evaluator\n",
    "from evoagentx.models import OpenAILLM, OpenAILLMConfig\n",
    "from evoagentx.optimizers import TextGradOptimizer\n",
    "from evoagentx.prompts import StringTemplate\n",
    "from evoagentx.workflow import SequentialWorkFlowGraph\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "from evoagentx.agents.agent_manager import AgentManager\n",
    "from evoagentx.benchmark import MBPP\n",
    "from evoagentx.core.callbacks import suppress_logger_info\n",
    "from evoagentx.core.logging import logger\n",
    "from evoagentx.evaluators import Evaluator\n",
    "from evoagentx.models import OpenAILLM, OpenAILLMConfig\n",
    "from evoagentx.optimizers import TextGradOptimizer\n",
    "from evoagentx.prompts import StringTemplate\n",
    "from evoagentx.workflow import SequentialWorkFlowGraph\n",
    "\n",
    "from evoagentx.models import OpenAILLMConfig, OpenAILLM\n",
    "from evoagentx.workflow import SEWWorkFlowGraph, STRUCTUREWorkFlowGraph\n",
    "from evoagentx.agents import AgentManager\n",
    "from evoagentx.benchmark import HumanEval,AFlowMBPP\n",
    "from evoagentx.evaluators import Evaluator \n",
    "from evoagentx.optimizers import SEWOptimizer, STRUCTUREOptimizer\n",
    "from evoagentx.optimizers.structure_optimizer import STRUCTUREWorkFlowScheme\n",
    "from evoagentx.core.callbacks import suppress_logger_info\n",
    "\n",
    "from evoagentx.models import OpenAILLMConfig, OpenAILLM,AzureOpenAIConfig,LiteLLMConfig,LiteLLM\n",
    "from evoagentx.workflow import SEWWorkFlowGraph \n",
    "from evoagentx.agents import AgentManager\n",
    "from evoagentx.benchmark import MBPPPLUS, AFlowMBPPPLUS\n",
    "from evoagentx.evaluators import Evaluator \n",
    "from evoagentx.optimizers import SEWOptimizer \n",
    "from evoagentx.core.callbacks import suppress_logger_info\n",
    "from evoagentx.benchmark import HumanEvalPLUS\n",
    "from evoagentx.benchmark import SciCode\n",
    "from evoagentx.benchmark import PertQA\n",
    "from copy import deepcopy\n",
    "\n",
    "import nest_asyncio\n",
    "nest_asyncio.apply()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "54fa1aa5",
   "metadata": {},
   "outputs": [],
   "source": [
    "from evoagentx.benchmark import PertQA\n",
    "from copy import deepcopy\n",
    "\n",
    "import nest_asyncio\n",
    "nest_asyncio.apply()\n",
    "\n",
    "def collate_func(example: dict) -> dict:\n",
    "    problem = \"Question: {}\\n\\nAnswer:\".format(example[\"question\"])\n",
    "    return {\"problem\": problem}\n",
    "\n",
    "\n",
    "hotpotqa_graph_data = {\n",
    "    \"goal\": \"Answer the question based on the context. The answer should be a direct response to the question, without including explanations or reasoning.\",\n",
    "    \"tasks\": [\n",
    "        {\n",
    "            \"name\": \"answer_generate\",\n",
    "            \"description\": \"Answer the question based on the context.\",\n",
    "            \"inputs\": [\n",
    "                {\"name\": \"problem\", \"type\": \"str\", \"required\": True, \"description\": \"The problem to solve.\"}\n",
    "            ],\n",
    "            \"outputs\": [\n",
    "                {\"name\": \"answer\", \"type\": \"str\", \"required\": True, \"description\": \"The answer to the problem.\"}\n",
    "            ],\n",
    "            \"prompt_template\": StringTemplate(instruction=\"Think step by step to answer the question. You should explain your thinking process in the 'thought' field, and provide the final answer in the 'answer' field.\\nFormat your output in xml format, such as <thought>xxx</thought> and <answer>xxx</answer>.\"),\n",
    "            \"parse_mode\": \"xml\"\n",
    "        }\n",
    "    ] \n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1ebace55",
   "metadata": {},
   "outputs": [],
   "source": [
    "api_key = \"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\"\n",
    "# Function to encode the image\n",
    "import os\n",
    "os.environ[\"OPENAI_API_KEY\"] = api_key\n",
    "OPENAI_API_KEY = os.getenv(\"OPENAI_API_KEY\")\n",
    "\n",
    "\n",
    "llm_config = OpenAILLMConfig(model=\"gpt-4o-mini-2024-07-18\", openai_key=OPENAI_API_KEY, top_p=0.85, temperature=0.2, frequency_penalty=0.0, presence_penalty=0.0)\n",
    "llm = OpenAILLM(config=llm_config)\n",
    "executor_llm = OpenAILLM(config=llm_config)\n",
    "optimizer_llm = OpenAILLM(config=llm_config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "20e078fa",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 00:51:11.467\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.benchmark.pertqa\u001b[0m:\u001b[36m_load_data_from_file\u001b[0m:\u001b[36m52\u001b[0m - \u001b[1mloading HotPotQA data from /home/tl688/pitl688/selfevolve/EvoAgentX/examples/pertqa/adamson_update_train.json ...\u001b[0m\n",
      "\u001b[32m2026-01-13 00:51:11.471\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.benchmark.pertqa\u001b[0m:\u001b[36m_load_data_from_file\u001b[0m:\u001b[36m52\u001b[0m - \u001b[1mloading HotPotQA data from /home/tl688/pitl688/selfevolve/EvoAgentX/examples/pertqa/adamson_update_train.json ...\u001b[0m\n",
      "\u001b[32m2026-01-13 00:51:11.474\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.benchmark.pertqa\u001b[0m:\u001b[36m_load_data_from_file\u001b[0m:\u001b[36m52\u001b[0m - \u001b[1mloading HotPotQA data from /home/tl688/pitl688/selfevolve/EvoAgentX/examples/pertqa/adamson_update_test.json ...\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "benchmark = PertQA(pertdata='adamson')\n",
    "workflow_graph = SequentialWorkFlowGraph.from_dict(hotpotqa_graph_data)\n",
    "agent_manager = AgentManager()\n",
    "agent_manager.add_agents_from_workflow(workflow_graph, executor_llm.config)\n",
    "\n",
    "evaluator = Evaluator(\n",
    "    llm=executor_llm, \n",
    "    agent_manager=agent_manager, \n",
    "    collate_func=collate_func, \n",
    "    num_workers=20, \n",
    "    verbose=True\n",
    ")\n",
    "\n",
    "textgrad_optimizer = TextGradOptimizer(\n",
    "    graph=workflow_graph, \n",
    "    optimize_mode=\"all\",\n",
    "    executor_llm=executor_llm, \n",
    "    optimizer_llm=optimizer_llm,\n",
    "    batch_size=3,\n",
    "    max_steps=20,\n",
    "    evaluator=evaluator,\n",
    "    eval_every_n_steps=1,\n",
    "    eval_rounds=1,\n",
    "    save_interval=None,\n",
    "    save_path=\"./\",\n",
    "    rollback=True,\n",
    "    constraints=[]\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b485d4ad",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2160"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(benchmark._dev_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "aa7522fd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3000"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(benchmark._test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "3b52c62d",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   0%|          | 1/3000 [00:04<3:50:03,  4.60s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   0%|          | 2/3000 [00:05<1:56:41,  2.34s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   0%|          | 4/3000 [00:05<43:31,  1.15it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   0%|          | 8/3000 [00:06<15:05,  3.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   0%|          | 10/3000 [00:06<11:55,  4.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   0%|          | 11/3000 [00:07<17:55,  2.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   0%|          | 15/3000 [00:07<10:33,  4.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|          | 19/3000 [00:08<08:29,  5.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|          | 22/3000 [00:09<12:23,  4.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|          | 26/3000 [00:09<06:59,  7.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 28/3000 [00:10<09:22,  5.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 30/3000 [00:10<11:22,  4.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 32/3000 [00:11<10:29,  4.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|          | 36/3000 [00:11<06:52,  7.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|▏         | 40/3000 [00:12<07:24,  6.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|▏         | 41/3000 [00:13<13:11,  3.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|▏         | 43/3000 [00:13<11:43,  4.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 45/3000 [00:13<11:00,  4.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 48/3000 [00:14<08:43,  5.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 49/3000 [00:14<11:54,  4.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 53/3000 [00:15<08:38,  5.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 56/3000 [00:15<06:30,  7.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 58/3000 [00:16<08:46,  5.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 60/3000 [00:16<11:20,  4.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 61/3000 [00:17<10:35,  4.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 63/3000 [00:17<10:57,  4.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 66/3000 [00:18<09:39,  5.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 67/3000 [00:18<11:11,  4.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 68/3000 [00:18<10:57,  4.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 71/3000 [00:19<11:36,  4.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▎         | 75/3000 [00:19<06:59,  6.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 77/3000 [00:20<08:52,  5.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 80/3000 [00:20<06:52,  7.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 84/3000 [00:21<09:19,  5.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 85/3000 [00:21<10:38,  4.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 87/3000 [00:22<10:09,  4.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 88/3000 [00:22<10:39,  4.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 91/3000 [00:23<08:48,  5.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 93/3000 [00:23<06:35,  7.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 95/3000 [00:23<07:47,  6.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 96/3000 [00:23<09:18,  5.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 97/3000 [00:24<10:28,  4.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 99/3000 [00:24<08:28,  5.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 102/3000 [00:24<07:20,  6.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 104/3000 [00:25<07:58,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▎         | 105/3000 [00:25<08:55,  5.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▎         | 106/3000 [00:25<11:14,  4.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▎         | 107/3000 [00:26<10:59,  4.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▎         | 108/3000 [00:26<13:40,  3.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▎         | 109/3000 [00:26<13:46,  3.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▎         | 111/3000 [00:27<12:10,  3.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 116/3000 [00:27<06:49,  7.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 119/3000 [00:28<07:10,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 121/3000 [00:28<06:21,  7.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 123/3000 [00:28<06:35,  7.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 124/3000 [00:29<08:22,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 127/3000 [00:30<16:07,  2.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 128/3000 [00:31<14:07,  3.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 129/3000 [00:31<13:54,  3.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 131/3000 [00:31<11:09,  4.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 133/3000 [00:31<08:11,  5.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 135/3000 [00:32<07:39,  6.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▍         | 138/3000 [00:32<06:21,  7.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 140/3000 [00:32<05:07,  9.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 142/3000 [00:32<05:38,  8.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 143/3000 [00:33<08:02,  5.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 144/3000 [00:33<09:04,  5.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 145/3000 [00:33<10:27,  4.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▍         | 147/3000 [00:34<14:15,  3.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▌         | 151/3000 [00:35<11:22,  4.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 153/3000 [00:36<11:58,  3.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▌         | 157/3000 [00:36<08:03,  5.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▌         | 162/3000 [00:37<05:18,  8.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 164/3000 [00:37<07:45,  6.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 166/3000 [00:38<11:29,  4.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 168/3000 [00:39<11:50,  3.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 169/3000 [00:39<12:31,  3.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 172/3000 [00:39<08:04,  5.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 175/3000 [00:40<08:39,  5.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 179/3000 [00:40<05:56,  7.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 181/3000 [00:41<07:29,  6.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 182/3000 [00:41<07:59,  5.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 183/3000 [00:41<09:12,  5.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 184/3000 [00:41<09:40,  4.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 186/3000 [00:42<11:38,  4.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▋         | 188/3000 [00:42<07:57,  5.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▋         | 189/3000 [00:43<09:01,  5.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▋         | 191/3000 [00:43<08:41,  5.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▋         | 193/3000 [00:43<09:03,  5.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 196/3000 [00:44<06:02,  7.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 198/3000 [00:44<07:00,  6.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 201/3000 [00:44<06:02,  7.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 202/3000 [00:45<11:55,  3.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 205/3000 [00:46<11:00,  4.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 206/3000 [00:46<09:58,  4.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 207/3000 [00:46<13:14,  3.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 209/3000 [00:47<11:05,  4.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 214/3000 [00:47<06:57,  6.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 215/3000 [00:48<09:27,  4.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 220/3000 [00:48<05:28,  8.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 221/3000 [00:48<06:10,  7.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 222/3000 [00:49<08:45,  5.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 223/3000 [00:49<13:33,  3.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 224/3000 [00:50<14:13,  3.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 226/3000 [00:50<13:13,  3.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 230/3000 [00:51<08:21,  5.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 233/3000 [00:51<08:51,  5.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 236/3000 [00:52<06:35,  6.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 239/3000 [00:52<06:18,  7.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 240/3000 [00:52<08:11,  5.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 241/3000 [00:53<10:13,  4.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 243/3000 [00:53<10:01,  4.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 247/3000 [00:54<08:00,  5.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 249/3000 [00:54<07:50,  5.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 250/3000 [00:54<07:18,  6.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 251/3000 [00:55<08:28,  5.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 254/3000 [00:55<07:52,  5.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▊         | 256/3000 [00:56<08:19,  5.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▊         | 257/3000 [00:56<13:09,  3.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▊         | 260/3000 [00:56<08:27,  5.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▊         | 261/3000 [00:57<10:43,  4.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 264/3000 [00:58<09:15,  4.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 265/3000 [00:58<08:15,  5.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 269/3000 [00:58<06:37,  6.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 270/3000 [00:58<06:58,  6.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 272/3000 [00:59<09:02,  5.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 273/3000 [00:59<12:15,  3.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 274/3000 [01:00<12:37,  3.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 275/3000 [01:00<12:21,  3.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 278/3000 [01:00<09:01,  5.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 279/3000 [01:01<09:39,  4.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 280/3000 [01:01<13:56,  3.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 284/3000 [01:02<08:20,  5.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|▉         | 287/3000 [01:02<06:40,  6.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|▉         | 289/3000 [01:03<08:29,  5.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 290/3000 [01:03<09:01,  5.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|▉         | 292/3000 [01:03<08:07,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 294/3000 [01:03<05:58,  7.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 296/3000 [01:04<06:09,  7.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 298/3000 [01:04<06:05,  7.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|█         | 301/3000 [01:05<11:45,  3.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 304/3000 [01:05<07:25,  6.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 306/3000 [01:06<11:17,  3.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 309/3000 [01:07<08:08,  5.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|█         | 312/3000 [01:07<08:04,  5.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 313/3000 [01:07<08:10,  5.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 316/3000 [01:08<06:29,  6.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 318/3000 [01:08<08:11,  5.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 319/3000 [01:09<12:52,  3.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 320/3000 [01:09<14:36,  3.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 322/3000 [01:10<10:31,  4.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 324/3000 [01:10<11:11,  3.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 326/3000 [01:10<07:30,  5.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 328/3000 [01:11<09:26,  4.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 333/3000 [01:11<06:23,  6.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 336/3000 [01:12<07:29,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█▏        | 338/3000 [01:12<07:28,  5.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█▏        | 339/3000 [01:13<07:22,  6.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█▏        | 342/3000 [01:13<06:34,  6.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█▏        | 344/3000 [01:14<09:29,  4.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 345/3000 [01:14<13:19,  3.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 346/3000 [01:14<12:17,  3.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 348/3000 [01:15<10:09,  4.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 349/3000 [01:15<09:54,  4.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 353/3000 [01:15<05:44,  7.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 356/3000 [01:16<07:12,  6.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 358/3000 [01:16<07:15,  6.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 361/3000 [01:17<11:04,  3.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 362/3000 [01:18<12:35,  3.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 364/3000 [01:18<10:26,  4.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 365/3000 [01:18<10:07,  4.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 367/3000 [01:19<08:39,  5.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 371/3000 [01:19<06:50,  6.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 373/3000 [01:19<06:03,  7.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 374/3000 [01:20<06:36,  6.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 377/3000 [01:20<06:55,  6.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 378/3000 [01:20<08:24,  5.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 379/3000 [01:21<09:19,  4.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 380/3000 [01:21<12:32,  3.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 382/3000 [01:22<10:27,  4.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 383/3000 [01:22<15:28,  2.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 388/3000 [01:23<07:01,  6.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 390/3000 [01:23<07:45,  5.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 393/3000 [01:24<07:06,  6.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 396/3000 [01:24<04:46,  9.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 398/3000 [01:25<09:34,  4.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 400/3000 [01:25<09:24,  4.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 401/3000 [01:25<08:55,  4.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 404/3000 [01:26<07:38,  5.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▎        | 406/3000 [01:27<12:22,  3.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▎        | 409/3000 [01:27<08:35,  5.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 415/3000 [01:27<05:05,  8.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 418/3000 [01:28<08:27,  5.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 420/3000 [01:29<07:50,  5.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 421/3000 [01:29<08:05,  5.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 422/3000 [01:29<09:51,  4.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 425/3000 [01:30<09:38,  4.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 427/3000 [01:30<08:40,  4.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 429/3000 [01:31<08:41,  4.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 430/3000 [01:31<08:33,  5.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 432/3000 [01:31<07:51,  5.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 433/3000 [01:32<09:10,  4.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 435/3000 [01:32<10:04,  4.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▍        | 437/3000 [01:33<13:55,  3.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 440/3000 [01:33<07:50,  5.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▍        | 443/3000 [01:34<06:48,  6.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▍        | 445/3000 [01:34<06:17,  6.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 446/3000 [01:34<10:16,  4.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 447/3000 [01:35<12:55,  3.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 450/3000 [01:36<10:34,  4.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 453/3000 [01:36<06:35,  6.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 455/3000 [01:36<05:16,  8.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 457/3000 [01:36<06:12,  6.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 459/3000 [01:37<08:07,  5.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 460/3000 [01:37<07:54,  5.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 461/3000 [01:37<08:45,  4.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 462/3000 [01:38<08:43,  4.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 463/3000 [01:38<11:08,  3.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 466/3000 [01:38<08:06,  5.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 467/3000 [01:39<10:26,  4.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 470/3000 [01:39<08:08,  5.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 471/3000 [01:40<11:28,  3.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 476/3000 [01:40<06:04,  6.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 479/3000 [01:41<07:17,  5.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 483/3000 [01:42<07:14,  5.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 485/3000 [01:42<10:44,  3.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▋        | 488/3000 [01:43<08:32,  4.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▋        | 490/3000 [01:43<08:22,  5.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▋        | 491/3000 [01:43<08:52,  4.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▋        | 493/3000 [01:44<08:50,  4.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▋        | 494/3000 [01:44<10:40,  3.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 496/3000 [01:45<12:31,  3.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 499/3000 [01:45<08:04,  5.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 501/3000 [01:46<08:12,  5.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 502/3000 [01:46<07:41,  5.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 504/3000 [01:46<06:42,  6.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 508/3000 [01:47<05:52,  7.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 509/3000 [01:47<09:19,  4.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 512/3000 [01:48<08:33,  4.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 513/3000 [01:48<09:15,  4.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 514/3000 [01:49<11:41,  3.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 515/3000 [01:49<11:22,  3.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 517/3000 [01:49<10:18,  4.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 518/3000 [01:49<09:09,  4.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 520/3000 [01:50<07:20,  5.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 523/3000 [01:50<06:26,  6.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 524/3000 [01:50<06:51,  6.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 527/3000 [01:51<05:46,  7.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 530/3000 [01:51<06:25,  6.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 531/3000 [01:51<06:12,  6.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 533/3000 [01:52<05:43,  7.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 534/3000 [01:52<11:51,  3.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 537/3000 [01:53<08:41,  4.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 538/3000 [01:53<08:00,  5.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 541/3000 [01:54<07:51,  5.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 544/3000 [01:54<07:08,  5.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 546/3000 [01:54<07:04,  5.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 547/3000 [01:55<08:31,  4.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 548/3000 [01:55<08:38,  4.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 550/3000 [01:55<09:15,  4.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 552/3000 [01:56<07:51,  5.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 553/3000 [01:56<08:12,  4.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 555/3000 [01:57<08:51,  4.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▊        | 557/3000 [01:57<07:22,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▊        | 559/3000 [01:57<05:47,  7.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▊        | 561/3000 [01:57<05:37,  7.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 563/3000 [01:58<05:51,  6.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 564/3000 [01:58<07:37,  5.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 566/3000 [01:58<06:45,  6.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 570/3000 [01:59<06:55,  5.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 571/3000 [01:59<06:29,  6.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 572/3000 [01:59<08:09,  4.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 575/3000 [02:00<08:38,  4.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 576/3000 [02:00<08:26,  4.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 577/3000 [02:01<08:46,  4.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 579/3000 [02:01<07:45,  5.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 581/3000 [02:01<07:02,  5.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 583/3000 [02:02<06:52,  5.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 585/3000 [02:02<07:16,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|█▉        | 588/3000 [02:02<05:58,  6.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 590/3000 [02:03<06:47,  5.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 591/3000 [02:04<11:11,  3.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 593/3000 [02:04<11:24,  3.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|█▉        | 595/3000 [02:05<09:39,  4.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|█▉        | 599/3000 [02:05<06:00,  6.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|██        | 601/3000 [02:05<06:22,  6.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 603/3000 [02:06<06:46,  5.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 605/3000 [02:06<08:00,  4.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 606/3000 [02:07<09:46,  4.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|██        | 608/3000 [02:07<09:27,  4.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 611/3000 [02:08<08:45,  4.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 612/3000 [02:08<10:32,  3.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 616/3000 [02:09<06:53,  5.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 618/3000 [02:09<06:40,  5.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 620/3000 [02:09<05:39,  7.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 621/3000 [02:10<07:52,  5.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 622/3000 [02:10<10:11,  3.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 624/3000 [02:10<07:55,  4.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 626/3000 [02:10<07:07,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 629/3000 [02:11<07:45,  5.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 632/3000 [02:12<09:02,  4.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 635/3000 [02:12<07:10,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 636/3000 [02:13<08:18,  4.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██▏       | 638/3000 [02:13<07:17,  5.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██▏       | 639/3000 [02:13<07:37,  5.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██▏       | 640/3000 [02:14<09:02,  4.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██▏       | 642/3000 [02:14<09:29,  4.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██▏       | 644/3000 [02:14<06:23,  6.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 647/3000 [02:15<07:24,  5.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 651/3000 [02:16<06:06,  6.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 652/3000 [02:16<10:27,  3.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 655/3000 [02:17<08:39,  4.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 657/3000 [02:17<07:26,  5.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 661/3000 [02:18<06:40,  5.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 666/3000 [02:18<04:37,  8.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 668/3000 [02:19<04:13,  9.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 670/3000 [02:19<04:27,  8.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 674/3000 [02:21<09:43,  3.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▎       | 675/3000 [02:21<09:16,  4.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 677/3000 [02:21<07:50,  4.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 679/3000 [02:22<07:56,  4.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 680/3000 [02:22<08:04,  4.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 683/3000 [02:22<08:26,  4.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 685/3000 [02:23<07:38,  5.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 686/3000 [02:23<09:04,  4.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 689/3000 [02:24<07:22,  5.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 690/3000 [02:24<07:57,  4.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 693/3000 [02:24<07:26,  5.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 695/3000 [02:25<05:50,  6.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 696/3000 [02:25<10:11,  3.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 699/3000 [02:26<07:49,  4.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 701/3000 [02:26<06:27,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 702/3000 [02:26<08:05,  4.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 704/3000 [02:27<09:49,  3.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▎       | 706/3000 [02:27<06:51,  5.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▎       | 707/3000 [02:28<08:07,  4.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▎       | 711/3000 [02:28<06:23,  5.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▎       | 712/3000 [02:29<08:00,  4.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 714/3000 [02:29<08:17,  4.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 715/3000 [02:30<11:49,  3.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 719/3000 [02:30<06:37,  5.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 722/3000 [02:30<05:05,  7.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 724/3000 [02:30<04:14,  8.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 726/3000 [02:31<06:49,  5.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 728/3000 [02:32<07:41,  4.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 732/3000 [02:32<06:11,  6.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 733/3000 [02:33<11:01,  3.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 734/3000 [02:33<10:25,  3.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 735/3000 [02:34<10:51,  3.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▍       | 738/3000 [02:34<06:40,  5.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 741/3000 [02:34<06:57,  5.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▍       | 744/3000 [02:35<06:21,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 745/3000 [02:35<08:30,  4.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▍       | 749/3000 [02:36<05:50,  6.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 750/3000 [02:36<08:06,  4.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▌       | 753/3000 [02:37<09:26,  3.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 755/3000 [02:37<07:09,  5.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 756/3000 [02:38<07:51,  4.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▌       | 758/3000 [02:38<07:47,  4.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 759/3000 [02:38<08:06,  4.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 760/3000 [02:39<08:21,  4.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▌       | 762/3000 [02:39<08:32,  4.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▌       | 764/3000 [02:40<08:19,  4.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 766/3000 [02:40<07:02,  5.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 767/3000 [02:40<06:04,  6.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 768/3000 [02:40<07:06,  5.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 770/3000 [02:41<07:27,  4.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 772/3000 [02:42<10:48,  3.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 777/3000 [02:42<04:41,  7.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 779/3000 [02:43<07:56,  4.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 780/3000 [02:43<07:13,  5.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 781/3000 [02:43<07:48,  4.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 784/3000 [02:44<06:51,  5.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 785/3000 [02:44<09:36,  3.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▋       | 788/3000 [02:45<08:11,  4.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▋       | 789/3000 [02:45<09:26,  3.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▋       | 792/3000 [02:45<07:11,  5.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▋       | 795/3000 [02:46<07:54,  4.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 796/3000 [02:47<10:45,  3.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 799/3000 [02:47<07:53,  4.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 800/3000 [02:48<09:15,  3.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 801/3000 [02:48<12:12,  3.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 807/3000 [02:49<06:00,  6.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 808/3000 [02:49<06:45,  5.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 811/3000 [02:49<05:54,  6.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 813/3000 [02:50<06:59,  5.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 814/3000 [02:50<07:58,  4.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 815/3000 [02:51<08:19,  4.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 816/3000 [02:51<10:07,  3.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 818/3000 [02:51<08:57,  4.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 823/3000 [02:52<04:55,  7.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 827/3000 [02:53<06:58,  5.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 830/3000 [02:53<05:08,  7.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 832/3000 [02:54<07:33,  4.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 833/3000 [02:54<07:38,  4.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 836/3000 [02:55<06:45,  5.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 838/3000 [02:55<08:32,  4.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 840/3000 [02:56<06:41,  5.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 843/3000 [02:56<04:08,  8.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 846/3000 [02:57<07:42,  4.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 848/3000 [02:57<07:58,  4.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 850/3000 [02:57<06:31,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 852/3000 [02:58<05:42,  6.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 853/3000 [02:59<12:02,  2.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 854/3000 [02:59<11:30,  3.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▊       | 856/3000 [03:00<10:08,  3.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▊       | 860/3000 [03:00<06:18,  5.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 863/3000 [03:01<06:38,  5.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 865/3000 [03:01<05:04,  7.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 867/3000 [03:01<05:17,  6.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 868/3000 [03:01<07:43,  4.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 870/3000 [03:02<08:33,  4.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 872/3000 [03:02<07:50,  4.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 873/3000 [03:03<07:59,  4.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 875/3000 [03:03<09:23,  3.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 877/3000 [03:04<07:33,  4.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 879/3000 [03:04<06:50,  5.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 882/3000 [03:04<05:52,  6.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 884/3000 [03:05<06:19,  5.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|██▉       | 889/3000 [03:05<03:21, 10.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 891/3000 [03:06<06:18,  5.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 893/3000 [03:06<06:12,  5.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 894/3000 [03:07<06:53,  5.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 895/3000 [03:07<09:37,  3.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|██▉       | 899/3000 [03:08<07:21,  4.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 900/3000 [03:08<07:11,  4.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 901/3000 [03:08<09:01,  3.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 902/3000 [03:09<09:21,  3.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 905/3000 [03:09<05:48,  6.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 906/3000 [03:09<06:22,  5.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 909/3000 [03:09<04:32,  7.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 910/3000 [03:10<05:31,  6.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|███       | 913/3000 [03:11<08:31,  4.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|███       | 915/3000 [03:12<10:04,  3.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 916/3000 [03:12<10:42,  3.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 919/3000 [03:12<07:15,  4.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 921/3000 [03:13<06:06,  5.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 924/3000 [03:13<05:08,  6.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 925/3000 [03:13<06:09,  5.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 928/3000 [03:14<05:54,  5.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 929/3000 [03:14<06:44,  5.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 932/3000 [03:15<09:50,  3.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 934/3000 [03:16<07:24,  4.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███▏      | 938/3000 [03:16<04:51,  7.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███▏      | 939/3000 [03:16<04:36,  7.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███▏      | 940/3000 [03:16<05:27,  6.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███▏      | 943/3000 [03:17<05:08,  6.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 946/3000 [03:17<05:48,  5.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 947/3000 [03:17<05:39,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 948/3000 [03:18<07:38,  4.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 951/3000 [03:19<07:05,  4.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 952/3000 [03:19<08:42,  3.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 953/3000 [03:19<08:18,  4.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 954/3000 [03:19<08:04,  4.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 955/3000 [03:20<07:51,  4.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 957/3000 [03:20<06:05,  5.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 960/3000 [03:20<06:19,  5.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 961/3000 [03:21<06:39,  5.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 966/3000 [03:22<06:16,  5.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 968/3000 [03:22<07:31,  4.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 971/3000 [03:23<07:37,  4.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▎      | 975/3000 [03:23<04:39,  7.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 980/3000 [03:24<04:09,  8.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 982/3000 [03:25<06:19,  5.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 984/3000 [03:25<08:20,  4.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 987/3000 [03:26<06:34,  5.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 989/3000 [03:26<06:39,  5.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 992/3000 [03:27<05:21,  6.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 993/3000 [03:27<07:27,  4.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 995/3000 [03:27<06:57,  4.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 996/3000 [03:28<06:45,  4.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 999/3000 [03:28<05:27,  6.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 1000/3000 [03:29<08:57,  3.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 1002/3000 [03:29<09:11,  3.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▎      | 1006/3000 [03:30<06:33,  5.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▎      | 1008/3000 [03:30<05:36,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▎      | 1011/3000 [03:31<04:16,  7.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 1013/3000 [03:31<05:31,  5.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1014/3000 [03:31<07:01,  4.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 1016/3000 [03:32<06:59,  4.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1017/3000 [03:32<07:16,  4.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1019/3000 [03:33<10:08,  3.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1020/3000 [03:33<09:37,  3.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 1024/3000 [03:34<05:28,  6.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 1026/3000 [03:34<06:00,  5.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1027/3000 [03:34<06:10,  5.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1029/3000 [03:35<05:12,  6.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1031/3000 [03:35<05:18,  6.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 1033/3000 [03:36<07:05,  4.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▍      | 1036/3000 [03:36<05:07,  6.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▍      | 1038/3000 [03:36<06:33,  4.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 1039/3000 [03:37<06:12,  5.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▍      | 1042/3000 [03:37<05:03,  6.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 1043/3000 [03:38<08:07,  4.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 1045/3000 [03:38<07:25,  4.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 1048/3000 [03:38<05:02,  6.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▌      | 1050/3000 [03:39<07:56,  4.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 1051/3000 [03:39<07:57,  4.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 1054/3000 [03:40<09:30,  3.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▌      | 1059/3000 [03:41<05:24,  5.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 1060/3000 [03:41<05:52,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 1063/3000 [03:41<04:57,  6.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 1064/3000 [03:42<06:29,  4.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1065/3000 [03:42<08:02,  4.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1066/3000 [03:43<09:01,  3.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      " {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1069/3000 [03:43<07:23,  4.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 1072/3000 [03:44<07:35,  4.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 1074/3000 [03:44<06:55,  4.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 1077/3000 [03:45<06:34,  4.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 1079/3000 [03:45<05:46,  5.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 1081/3000 [03:46<05:48,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 1084/3000 [03:46<05:16,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 1086/3000 [03:47<07:07,  4.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1087/3000 [03:47<07:26,  4.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▋      | 1090/3000 [03:48<05:49,  5.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▋      | 1093/3000 [03:48<05:52,  5.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▋      | 1094/3000 [03:49<05:40,  5.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 1097/3000 [03:50<07:57,  3.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1098/3000 [03:50<07:42,  4.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1100/3000 [03:50<06:29,  4.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 1102/3000 [03:50<06:01,  5.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1105/3000 [03:51<04:44,  6.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1108/3000 [03:51<04:50,  6.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1109/3000 [03:52<06:16,  5.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 1113/3000 [03:52<04:38,  6.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1114/3000 [03:52<05:53,  5.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1115/3000 [03:53<11:17,  2.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 1119/3000 [03:54<05:42,  5.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 1123/3000 [03:54<04:25,  7.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1127/3000 [03:55<04:01,  7.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1129/3000 [03:55<05:08,  6.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1132/3000 [03:56<04:55,  6.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1134/3000 [03:56<06:56,  4.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1135/3000 [03:57<09:38,  3.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1137/3000 [03:58<12:28,  2.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1141/3000 [03:59<06:16,  4.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1144/3000 [03:59<04:15,  7.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1146/3000 [03:59<04:02,  7.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1148/3000 [04:00<04:41,  6.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1150/3000 [04:00<05:22,  5.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1152/3000 [04:00<04:07,  7.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1154/3000 [04:01<07:39,  4.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1155/3000 [04:02<10:11,  3.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▊      | 1159/3000 [04:02<06:22,  4.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▊      | 1160/3000 [04:02<06:36,  4.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▊      | 1162/3000 [04:03<06:10,  4.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 1166/3000 [04:03<05:13,  5.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1167/3000 [04:04<05:58,  5.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1170/3000 [04:04<04:15,  7.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 1174/3000 [04:05<03:58,  7.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1175/3000 [04:05<07:14,  4.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 1177/3000 [04:06<07:22,  4.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1179/3000 [04:06<05:11,  5.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1180/3000 [04:07<09:03,  3.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1182/3000 [04:07<07:03,  4.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 1185/3000 [04:07<05:06,  5.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|███▉      | 1189/3000 [04:08<04:02,  7.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 1191/3000 [04:08<05:32,  5.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 1193/3000 [04:09<06:48,  4.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 1194/3000 [04:09<06:44,  4.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|███▉      | 1197/3000 [04:10<07:13,  4.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|███▉      | 1199/3000 [04:10<06:22,  4.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|████      | 1202/3000 [04:11<06:34,  4.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1203/3000 [04:12<07:44,  3.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1205/3000 [04:12<06:42,  4.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1209/3000 [04:12<04:59,  5.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1211/3000 [04:13<04:50,  6.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1212/3000 [04:13<05:07,  5.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1213/3000 [04:14<07:55,  3.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1216/3000 [04:15<09:48,  3.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1218/3000 [04:15<08:42,  3.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1220/3000 [04:16<07:49,  3.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1221/3000 [04:16<08:44,  3.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1222/3000 [04:16<09:29,  3.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1225/3000 [04:17<07:04,  4.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1229/3000 [04:17<04:29,  6.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1230/3000 [04:18<04:32,  6.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1233/3000 [04:18<04:22,  6.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1234/3000 [04:18<04:08,  7.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1236/3000 [04:19<07:56,  3.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1237/3000 [04:19<07:44,  3.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████▏     | 1239/3000 [04:20<09:02,  3.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████▏     | 1241/3000 [04:20<07:28,  3.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1245/3000 [04:21<04:08,  7.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1247/3000 [04:21<05:34,  5.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1251/3000 [04:22<05:07,  5.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1255/3000 [04:23<04:14,  6.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1257/3000 [04:23<06:45,  4.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1258/3000 [04:24<06:48,  4.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1259/3000 [04:24<10:34,  2.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1261/3000 [04:25<07:42,  3.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1264/3000 [04:25<06:13,  4.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1265/3000 [04:25<05:44,  5.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1268/3000 [04:26<04:07,  6.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1272/3000 [04:26<03:46,  7.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1273/3000 [04:26<04:02,  7.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▎     | 1275/3000 [04:27<04:42,  6.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1276/3000 [04:27<05:19,  5.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1277/3000 [04:27<05:35,  5.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1278/3000 [04:28<13:14,  2.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1280/3000 [04:29<10:53,  2.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1282/3000 [04:29<08:23,  3.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1284/3000 [04:30<07:32,  3.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1287/3000 [04:30<06:11,  4.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1289/3000 [04:31<05:06,  5.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1292/3000 [04:31<04:39,  6.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1293/3000 [04:32<06:35,  4.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1295/3000 [04:32<05:39,  5.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1296/3000 [04:33<08:45,  3.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1300/3000 [04:33<06:25,  4.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1301/3000 [04:34<06:33,  4.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1303/3000 [04:34<06:34,  4.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1304/3000 [04:34<05:41,  4.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▎     | 1307/3000 [04:35<05:08,  5.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▎     | 1308/3000 [04:35<06:14,  4.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▎     | 1311/3000 [04:36<05:28,  5.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1313/3000 [04:36<07:25,  3.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1314/3000 [04:37<08:48,  3.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1316/3000 [04:37<07:12,  3.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1319/3000 [04:38<05:54,  4.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1323/3000 [04:38<03:55,  7.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1324/3000 [04:38<04:40,  5.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1325/3000 [04:39<05:50,  4.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1328/3000 [04:39<05:04,  5.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1330/3000 [04:40<05:22,  5.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1332/3000 [04:40<04:27,  6.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1333/3000 [04:41<09:04,  3.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1336/3000 [04:42<09:25,  2.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▍     | 1339/3000 [04:42<06:49,  4.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1341/3000 [04:42<05:02,  5.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▍     | 1344/3000 [04:43<04:37,  5.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1345/3000 [04:43<04:15,  6.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1346/3000 [04:43<05:19,  5.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▌     | 1351/3000 [04:44<03:02,  9.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1353/3000 [04:45<08:10,  3.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▌     | 1356/3000 [04:46<06:47,  4.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▌     | 1359/3000 [04:46<06:42,  4.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1361/3000 [04:47<04:48,  5.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▌     | 1364/3000 [04:47<04:30,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1366/3000 [04:47<04:27,  6.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1369/3000 [04:47<02:47,  9.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1371/3000 [04:49<06:39,  4.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1373/3000 [04:49<07:19,  3.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1374/3000 [04:49<06:50,  3.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1376/3000 [04:50<06:27,  4.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1377/3000 [04:50<05:42,  4.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1379/3000 [04:51<06:12,  4.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1381/3000 [04:51<04:51,  5.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1384/3000 [04:51<03:31,  7.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▋     | 1388/3000 [04:51<03:05,  8.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▋     | 1390/3000 [04:52<05:54,  4.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▋     | 1392/3000 [04:53<06:51,  3.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▋     | 1394/3000 [04:54<06:51,  3.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1397/3000 [04:54<04:35,  5.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1399/3000 [04:54<04:13,  6.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1401/3000 [04:54<03:10,  8.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1403/3000 [04:54<03:14,  8.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1405/3000 [04:55<05:23,  4.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1406/3000 [04:56<06:10,  4.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1407/3000 [04:56<06:15,  4.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1408/3000 [04:56<07:37,  3.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1411/3000 [04:57<06:16,  4.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1415/3000 [04:58<05:41,  4.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1416/3000 [04:58<05:25,  4.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1417/3000 [04:58<06:07,  4.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1423/3000 [04:59<03:48,  6.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1425/3000 [05:00<05:51,  4.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1428/3000 [05:00<06:13,  4.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1430/3000 [05:01<05:56,  4.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1433/3000 [05:01<04:49,  5.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1434/3000 [05:01<04:51,  5.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1437/3000 [05:02<04:55,  5.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1439/3000 [05:02<05:10,  5.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1441/3000 [05:03<04:55,  5.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1443/3000 [05:03<03:41,  7.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1444/3000 [05:03<04:02,  6.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1445/3000 [05:03<04:36,  5.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1446/3000 [05:04<05:08,  5.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1448/3000 [05:04<05:12,  4.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1450/3000 [05:05<06:13,  4.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1452/3000 [05:05<05:31,  4.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1453/3000 [05:05<05:40,  4.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1455/3000 [05:06<05:11,  4.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▊     | 1456/3000 [05:06<06:30,  3.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▊     | 1460/3000 [05:07<04:13,  6.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▊     | 1461/3000 [05:07<05:00,  5.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▊     | 1462/3000 [05:07<05:04,  5.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1465/3000 [05:08<06:19,  4.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1466/3000 [05:08<06:09,  4.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1467/3000 [05:09<06:09,  4.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1471/3000 [05:09<04:41,  5.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1473/3000 [05:09<03:42,  6.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1474/3000 [05:10<04:13,  6.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1476/3000 [05:10<04:35,  5.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1478/3000 [05:10<04:46,  5.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1481/3000 [05:11<05:59,  4.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1483/3000 [05:12<07:42,  3.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1485/3000 [05:12<05:34,  4.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1486/3000 [05:13<05:50,  4.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1488/3000 [05:13<04:34,  5.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1489/3000 [05:13<04:43,  5.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|████▉     | 1491/3000 [05:14<05:21,  4.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|████▉     | 1494/3000 [05:14<03:44,  6.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1495/3000 [05:14<04:41,  5.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1497/3000 [05:15<06:20,  3.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1498/3000 [05:15<06:04,  4.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1499/3000 [05:16<07:44,  3.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1501/3000 [05:16<05:53,  4.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 1503/3000 [05:17<06:13,  4.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1505/3000 [05:17<04:38,  5.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1506/3000 [05:17<06:26,  3.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1508/3000 [05:17<05:02,  4.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 1511/3000 [05:18<04:29,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1513/3000 [05:18<04:03,  6.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1516/3000 [05:19<04:13,  5.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1517/3000 [05:19<05:59,  4.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1519/3000 [05:20<05:19,  4.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1520/3000 [05:20<06:43,  3.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1522/3000 [05:21<06:26,  3.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1524/3000 [05:21<04:23,  5.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1526/3000 [05:21<04:32,  5.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1527/3000 [05:21<04:28,  5.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1528/3000 [05:22<04:37,  5.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1530/3000 [05:22<04:16,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1531/3000 [05:23<07:35,  3.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1536/3000 [05:23<04:37,  5.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████▏    | 1538/3000 [05:24<06:30,  3.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████▏    | 1542/3000 [05:24<03:31,  6.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████▏    | 1544/3000 [05:25<04:40,  5.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1547/3000 [05:26<04:03,  5.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1549/3000 [05:26<03:16,  7.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1551/3000 [05:26<04:25,  5.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1552/3000 [05:27<05:35,  4.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1554/3000 [05:27<05:00,  4.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1555/3000 [05:27<04:33,  5.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1556/3000 [05:28<05:52,  4.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1557/3000 [05:28<05:45,  4.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1560/3000 [05:28<04:49,  4.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1562/3000 [05:29<04:17,  5.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1565/3000 [05:29<04:04,  5.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1567/3000 [05:30<03:57,  6.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1569/3000 [05:30<03:27,  6.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1570/3000 [05:30<03:51,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1571/3000 [05:30<04:05,  5.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1573/3000 [05:31<05:20,  4.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1574/3000 [05:31<04:48,  4.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1576/3000 [05:32<06:06,  3.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1577/3000 [05:32<06:04,  3.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1580/3000 [05:32<04:22,  5.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1581/3000 [05:33<04:52,  4.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1584/3000 [05:33<03:57,  5.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1585/3000 [05:33<05:08,  4.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1586/3000 [05:34<05:40,  4.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1589/3000 [05:34<05:09,  4.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1592/3000 [05:35<04:31,  5.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1595/3000 [05:36<05:41,  4.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1596/3000 [05:36<05:38,  4.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1598/3000 [05:37<05:08,  4.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1600/3000 [05:37<05:29,  4.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1602/3000 [05:37<03:58,  5.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1604/3000 [05:38<05:56,  3.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▎    | 1606/3000 [05:38<04:26,  5.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▎    | 1609/3000 [05:39<03:17,  7.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▎    | 1610/3000 [05:39<04:44,  4.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▎    | 1611/3000 [05:39<05:45,  4.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▎    | 1612/3000 [05:40<05:36,  4.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1617/3000 [05:40<03:05,  7.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1618/3000 [05:41<06:26,  3.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1619/3000 [05:41<06:04,  3.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1621/3000 [05:42<07:18,  3.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1625/3000 [05:42<04:26,  5.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1629/3000 [05:43<02:37,  8.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1631/3000 [05:43<04:05,  5.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1633/3000 [05:44<04:14,  5.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1636/3000 [05:44<02:58,  7.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1638/3000 [05:45<06:33,  3.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▍    | 1640/3000 [05:46<06:09,  3.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▍    | 1644/3000 [05:46<03:46,  5.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1646/3000 [05:47<04:53,  4.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1648/3000 [05:47<04:31,  4.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1650/3000 [05:48<04:48,  4.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▌    | 1654/3000 [05:48<03:32,  6.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1656/3000 [05:48<03:45,  5.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1657/3000 [05:49<04:30,  4.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1658/3000 [05:50<08:46,  2.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▌    | 1660/3000 [05:50<06:40,  3.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▌    | 1664/3000 [05:51<04:58,  4.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1666/3000 [05:51<03:55,  5.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1667/3000 [05:52<04:23,  5.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1670/3000 [05:52<04:25,  5.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1674/3000 [05:52<02:32,  8.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1676/3000 [05:53<03:24,  6.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1677/3000 [05:53<04:07,  5.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1678/3000 [05:54<04:39,  4.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1679/3000 [05:54<07:37,  2.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1684/3000 [05:55<04:03,  5.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1685/3000 [05:55<03:45,  5.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1686/3000 [05:55<04:13,  5.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▋    | 1688/3000 [05:56<04:10,  5.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▋    | 1690/3000 [05:56<03:55,  5.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▋    | 1692/3000 [05:56<04:08,  5.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▋    | 1695/3000 [05:57<03:22,  6.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1696/3000 [05:57<05:14,  4.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1698/3000 [05:58<05:05,  4.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1700/3000 [05:58<04:49,  4.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1702/3000 [05:59<05:23,  4.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1705/3000 [05:59<03:23,  6.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1707/3000 [06:00<03:45,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1709/3000 [06:00<04:28,  4.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1712/3000 [06:01<05:05,  4.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1714/3000 [06:01<05:19,  4.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1715/3000 [06:02<04:32,  4.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1716/3000 [06:02<04:41,  4.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1717/3000 [06:02<06:22,  3.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1720/3000 [06:03<04:24,  4.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1723/3000 [06:03<04:09,  5.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▊    | 1725/3000 [06:04<03:11,  6.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1729/3000 [06:04<02:42,  7.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1730/3000 [06:04<02:50,  7.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1732/3000 [06:05<04:18,  4.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1734/3000 [06:05<03:23,  6.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1735/3000 [06:05<04:03,  5.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1736/3000 [06:06<04:28,  4.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1738/3000 [06:06<05:36,  3.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1740/3000 [06:07<03:53,  5.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1742/3000 [06:07<03:59,  5.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1744/3000 [06:07<03:02,  6.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1748/3000 [06:08<03:56,  5.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1752/3000 [06:09<03:24,  6.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1753/3000 [06:09<04:09,  5.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1755/3000 [06:10<05:17,  3.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▊    | 1756/3000 [06:10<06:04,  3.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▊    | 1757/3000 [06:11<06:31,  3.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▊    | 1761/3000 [06:11<03:50,  5.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1764/3000 [06:11<02:57,  6.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1767/3000 [06:12<03:12,  6.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1769/3000 [06:12<02:35,  7.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1771/3000 [06:13<04:06,  4.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1773/3000 [06:13<04:06,  4.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1776/3000 [06:14<03:33,  5.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1777/3000 [06:14<04:34,  4.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1779/3000 [06:15<05:33,  3.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1781/3000 [06:15<04:48,  4.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1783/3000 [06:15<03:29,  5.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1785/3000 [06:16<03:12,  6.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1786/3000 [06:16<04:12,  4.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|█████▉    | 1790/3000 [06:17<03:08,  6.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1792/3000 [06:17<02:45,  7.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|█████▉    | 1794/3000 [06:17<03:06,  6.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1796/3000 [06:18<03:53,  5.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|█████▉    | 1799/3000 [06:19<04:41,  4.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|██████    | 1802/3000 [06:19<04:27,  4.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1805/3000 [06:19<02:42,  7.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|██████    | 1809/3000 [06:20<02:32,  7.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1811/3000 [06:20<03:12,  6.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1812/3000 [06:21<03:27,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1813/3000 [06:21<03:36,  5.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1815/3000 [06:21<03:46,  5.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1816/3000 [06:22<04:57,  3.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1817/3000 [06:22<06:33,  3.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1818/3000 [06:23<06:02,  3.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1821/3000 [06:23<04:00,  4.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1824/3000 [06:23<02:37,  7.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1825/3000 [06:23<02:57,  6.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1827/3000 [06:24<03:16,  5.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1831/3000 [06:24<02:30,  7.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1835/3000 [06:25<02:57,  6.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1837/3000 [06:26<04:44,  4.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████▏   | 1838/3000 [06:26<04:46,  4.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████▏   | 1839/3000 [06:26<04:39,  4.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████▏   | 1842/3000 [06:27<03:34,  5.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████▏   | 1843/3000 [06:27<04:14,  4.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████▏   | 1844/3000 [06:28<05:07,  3.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1849/3000 [06:28<03:11,  6.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1851/3000 [06:28<02:45,  6.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1852/3000 [06:29<02:39,  7.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1853/3000 [06:29<03:14,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1855/3000 [06:29<04:00,  4.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1857/3000 [06:30<03:24,  5.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1858/3000 [06:30<03:17,  5.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1860/3000 [06:31<05:01,  3.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1862/3000 [06:31<05:10,  3.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1868/3000 [06:32<02:21,  8.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1871/3000 [06:32<03:01,  6.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1872/3000 [06:33<03:01,  6.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1874/3000 [06:33<03:51,  4.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▎   | 1875/3000 [06:34<05:30,  3.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1876/3000 [06:34<06:10,  3.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1879/3000 [06:35<04:36,  4.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1881/3000 [06:35<03:20,  5.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1883/3000 [06:36<04:07,  4.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1884/3000 [06:36<03:45,  4.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1888/3000 [06:36<02:30,  7.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1894/3000 [06:37<01:59,  9.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1896/3000 [06:38<03:37,  5.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1897/3000 [06:39<05:59,  3.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1898/3000 [06:39<05:47,  3.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1902/3000 [06:39<03:26,  5.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1903/3000 [06:40<04:53,  3.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▎   | 1907/3000 [06:40<02:58,  6.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▎   | 1909/3000 [06:41<03:09,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▎   | 1912/3000 [06:41<02:03,  8.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1915/3000 [06:41<03:03,  5.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1916/3000 [06:42<03:18,  5.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1917/3000 [06:42<04:48,  3.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1919/3000 [06:43<05:43,  3.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1921/3000 [06:43<03:41,  4.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1923/3000 [06:44<03:27,  5.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1924/3000 [06:44<03:52,  4.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1928/3000 [06:44<02:30,  7.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1932/3000 [06:44<01:45, 10.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1934/3000 [06:45<02:52,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▍   | 1936/3000 [06:45<03:14,  5.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▍   | 1938/3000 [06:46<03:48,  4.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1939/3000 [06:46<03:44,  4.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1940/3000 [06:46<03:44,  4.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▍   | 1944/3000 [06:47<02:47,  6.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1946/3000 [06:47<02:15,  7.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▍   | 1949/3000 [06:48<02:19,  7.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1951/3000 [06:48<02:31,  6.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1952/3000 [06:48<02:59,  5.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1953/3000 [06:49<03:55,  4.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1954/3000 [06:49<03:54,  4.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1956/3000 [06:49<03:07,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 1958/3000 [06:50<04:10,  4.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 1961/3000 [06:50<02:35,  6.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 1963/3000 [06:50<02:24,  7.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1966/3000 [06:51<01:58,  8.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1967/3000 [06:51<02:38,  6.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1969/3000 [06:51<02:29,  6.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1973/3000 [06:53<04:30,  3.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1975/3000 [06:53<03:51,  4.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1977/3000 [06:54<03:36,  4.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1979/3000 [06:54<02:56,  5.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1982/3000 [06:54<02:08,  7.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1983/3000 [06:54<02:20,  7.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1986/3000 [06:55<02:10,  7.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▋   | 1989/3000 [06:55<03:12,  5.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▋   | 1990/3000 [06:56<04:34,  3.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▋   | 1991/3000 [06:56<05:07,  3.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▋   | 1992/3000 [06:57<05:40,  2.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▋   | 1995/3000 [06:57<03:58,  4.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 1997/3000 [06:58<03:08,  5.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2004/3000 [06:58<01:19, 12.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2006/3000 [06:58<01:42,  9.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2008/3000 [06:59<02:42,  6.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2010/3000 [07:00<04:32,  3.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 2012/3000 [07:00<03:56,  4.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 2014/3000 [07:01<03:07,  5.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2016/3000 [07:01<02:21,  6.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 2019/3000 [07:02<02:49,  5.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2022/3000 [07:02<02:10,  7.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2024/3000 [07:02<01:53,  8.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2026/3000 [07:02<02:32,  6.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2027/3000 [07:03<03:04,  5.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2029/3000 [07:03<03:46,  4.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2030/3000 [07:04<04:10,  3.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2033/3000 [07:04<03:17,  4.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2034/3000 [07:05<03:28,  4.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2037/3000 [07:05<02:50,  5.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2038/3000 [07:05<03:30,  4.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2039/3000 [07:06<03:42,  4.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2041/3000 [07:06<03:40,  4.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2045/3000 [07:06<02:05,  7.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2047/3000 [07:07<02:19,  6.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2048/3000 [07:07<02:44,  5.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2049/3000 [07:07<03:36,  4.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2052/3000 [07:08<02:27,  6.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2053/3000 [07:08<04:01,  3.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2054/3000 [07:09<04:07,  3.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▊   | 2056/3000 [07:09<04:26,  3.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▊   | 2060/3000 [07:09<02:05,  7.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2063/3000 [07:10<02:00,  7.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2065/3000 [07:10<01:56,  8.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2067/3000 [07:11<02:24,  6.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2069/3000 [07:11<02:42,  5.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 2072/3000 [07:11<02:28,  6.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2074/3000 [07:13<04:27,  3.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2076/3000 [07:13<04:46,  3.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 2080/3000 [07:14<02:57,  5.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2081/3000 [07:14<03:00,  5.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2082/3000 [07:14<03:04,  4.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|██████▉   | 2085/3000 [07:15<02:46,  5.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 2087/3000 [07:15<02:35,  5.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 2088/3000 [07:15<03:15,  4.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 2089/3000 [07:15<03:23,  4.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 2092/3000 [07:16<02:14,  6.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|██████▉   | 2094/3000 [07:16<03:20,  4.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 2096/3000 [07:17<02:41,  5.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|██████▉   | 2099/3000 [07:17<02:31,  5.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|███████   | 2102/3000 [07:18<02:36,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 2103/3000 [07:18<02:31,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|███████   | 2106/3000 [07:19<03:32,  4.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 2107/3000 [07:19<03:15,  4.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 2108/3000 [07:20<04:20,  3.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 2110/3000 [07:20<03:53,  3.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 2112/3000 [07:20<03:16,  4.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 2116/3000 [07:21<02:13,  6.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2118/3000 [07:21<02:00,  7.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2119/3000 [07:22<03:21,  4.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2120/3000 [07:22<03:28,  4.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2122/3000 [07:22<03:11,  4.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2124/3000 [07:23<03:37,  4.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 2127/3000 [07:24<03:15,  4.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2128/3000 [07:24<02:55,  4.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2131/3000 [07:24<02:24,  6.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 2135/3000 [07:25<01:50,  7.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2137/3000 [07:25<02:37,  5.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████▏  | 2138/3000 [07:25<02:55,  4.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████▏  | 2142/3000 [07:26<02:08,  6.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████▏  | 2143/3000 [07:26<02:49,  5.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2146/3000 [07:27<02:56,  4.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2148/3000 [07:27<02:45,  5.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2150/3000 [07:28<02:45,  5.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2151/3000 [07:28<02:35,  5.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2154/3000 [07:28<02:09,  6.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2155/3000 [07:29<04:00,  3.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2156/3000 [07:29<04:10,  3.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2159/3000 [07:29<02:34,  5.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2160/3000 [07:30<02:54,  4.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2163/3000 [07:30<02:31,  5.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2165/3000 [07:31<03:01,  4.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2166/3000 [07:31<03:37,  3.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2167/3000 [07:31<03:29,  3.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2169/3000 [07:32<04:18,  3.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2172/3000 [07:33<02:32,  5.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▎  | 2175/3000 [07:33<02:12,  6.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 2178/3000 [07:33<01:44,  7.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2179/3000 [07:34<02:02,  6.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2180/3000 [07:34<02:55,  4.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2183/3000 [07:34<02:06,  6.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2184/3000 [07:35<02:30,  5.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2185/3000 [07:35<03:55,  3.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2187/3000 [07:36<04:08,  3.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 2189/3000 [07:36<03:18,  4.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 2191/3000 [07:37<02:59,  4.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2194/3000 [07:37<01:47,  7.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 2197/3000 [07:37<02:03,  6.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2198/3000 [07:38<02:28,  5.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2200/3000 [07:38<02:20,  5.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2201/3000 [07:38<02:28,  5.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2203/3000 [07:39<02:47,  4.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▎  | 2206/3000 [07:39<02:09,  6.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▎  | 2207/3000 [07:40<03:42,  3.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▎  | 2209/3000 [07:40<03:10,  4.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▎  | 2211/3000 [07:40<02:37,  5.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2213/3000 [07:41<02:22,  5.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 2216/3000 [07:41<02:35,  5.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 2219/3000 [07:42<01:56,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2220/3000 [07:42<01:53,  6.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2221/3000 [07:42<02:39,  4.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2222/3000 [07:43<03:39,  3.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2225/3000 [07:43<02:19,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2227/3000 [07:44<02:59,  4.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2228/3000 [07:44<03:04,  4.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 2230/3000 [07:44<02:55,  4.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2233/3000 [07:44<01:41,  7.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2235/3000 [07:45<01:53,  6.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▍  | 2237/3000 [07:45<02:14,  5.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2238/3000 [07:45<02:18,  5.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2239/3000 [07:46<02:51,  4.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▍  | 2241/3000 [07:46<03:17,  3.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2242/3000 [07:47<03:08,  4.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2244/3000 [07:47<03:40,  3.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2245/3000 [07:48<04:01,  3.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▍  | 2247/3000 [07:48<03:11,  3.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 2250/3000 [07:48<01:55,  6.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 2251/3000 [07:49<02:42,  4.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▌  | 2255/3000 [07:49<02:02,  6.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 2257/3000 [07:49<01:34,  7.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 2259/3000 [07:50<02:19,  5.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 2260/3000 [07:50<02:22,  5.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 2264/3000 [07:51<02:01,  6.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2265/3000 [07:51<02:43,  4.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2266/3000 [07:52<03:05,  3.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2269/3000 [07:52<02:50,  4.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 2274/3000 [07:53<01:43,  6.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2276/3000 [07:53<01:32,  7.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 2279/3000 [07:54<01:56,  6.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2280/3000 [07:54<02:10,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 2282/3000 [07:54<02:29,  4.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2283/3000 [07:54<02:11,  5.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2284/3000 [07:55<04:26,  2.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▋  | 2288/3000 [07:56<02:20,  5.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▋  | 2289/3000 [07:56<02:30,  4.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 2296/3000 [07:57<01:16,  9.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2298/3000 [07:57<01:08, 10.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2300/3000 [07:58<02:44,  4.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2302/3000 [07:58<02:48,  4.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 2304/3000 [07:59<02:28,  4.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2306/3000 [07:59<02:33,  4.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2307/3000 [08:00<02:44,  4.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2309/3000 [08:00<02:34,  4.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2313/3000 [08:00<01:33,  7.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 2316/3000 [08:01<01:32,  7.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2317/3000 [08:01<01:30,  7.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2318/3000 [08:01<01:48,  6.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2319/3000 [08:02<02:46,  4.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2320/3000 [08:02<03:08,  3.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2322/3000 [08:02<02:39,  4.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2325/3000 [08:03<02:16,  4.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2328/3000 [08:03<02:14,  5.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2329/3000 [08:04<02:41,  4.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2334/3000 [08:04<01:45,  6.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2337/3000 [08:05<01:31,  7.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2338/3000 [08:05<01:38,  6.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2339/3000 [08:05<01:56,  5.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2340/3000 [08:06<02:14,  4.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2344/3000 [08:06<01:59,  5.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2345/3000 [08:07<02:05,  5.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2348/3000 [08:07<02:33,  4.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2350/3000 [08:08<02:29,  4.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2353/3000 [08:08<01:49,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2355/3000 [08:09<02:02,  5.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▊  | 2361/3000 [08:09<01:12,  8.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2363/3000 [08:10<01:26,  7.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2365/3000 [08:11<02:28,  4.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2367/3000 [08:11<01:59,  5.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 2372/3000 [08:11<01:35,  6.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 2378/3000 [08:13<01:38,  6.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2380/3000 [08:13<01:37,  6.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 2384/3000 [08:14<02:18,  4.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|███████▉  | 2387/3000 [08:15<02:00,  5.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 2388/3000 [08:15<01:52,  5.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 2390/3000 [08:15<01:41,  5.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 2392/3000 [08:16<02:07,  4.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|███████▉  | 2396/3000 [08:17<01:40,  6.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 2398/3000 [08:17<01:21,  7.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2400/3000 [08:18<02:49,  3.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 2403/3000 [08:18<02:12,  4.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 2405/3000 [08:19<01:50,  5.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2408/3000 [08:19<01:35,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2410/3000 [08:19<01:33,  6.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 2413/3000 [08:20<01:24,  6.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2414/3000 [08:20<01:49,  5.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2417/3000 [08:21<01:48,  5.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2418/3000 [08:21<01:58,  4.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2420/3000 [08:21<01:53,  5.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2422/3000 [08:22<01:43,  5.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2424/3000 [08:22<01:55,  4.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2428/3000 [08:23<01:26,  6.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2430/3000 [08:23<01:28,  6.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2431/3000 [08:23<01:34,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2432/3000 [08:24<02:01,  4.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2433/3000 [08:24<02:09,  4.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2434/3000 [08:24<02:25,  3.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2436/3000 [08:25<01:52,  5.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2437/3000 [08:25<02:03,  4.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████▏ | 2438/3000 [08:25<02:04,  4.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████▏ | 2439/3000 [08:25<02:24,  3.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████▏ | 2440/3000 [08:26<02:18,  4.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████▏ | 2442/3000 [08:26<01:54,  4.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2445/3000 [08:26<01:32,  5.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2447/3000 [08:27<01:15,  7.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2450/3000 [08:27<01:11,  7.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2452/3000 [08:27<01:28,  6.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2453/3000 [08:28<01:45,  5.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2454/3000 [08:28<01:47,  5.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2456/3000 [08:28<01:30,  6.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2458/3000 [08:29<02:06,  4.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2461/3000 [08:29<01:19,  6.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2462/3000 [08:29<01:28,  6.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2465/3000 [08:30<01:34,  5.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2467/3000 [08:30<01:17,  6.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2468/3000 [08:30<01:36,  5.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2469/3000 [08:31<01:53,  4.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2471/3000 [08:31<01:57,  4.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2473/3000 [08:32<01:50,  4.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2474/3000 [08:32<01:38,  5.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2476/3000 [08:32<01:45,  4.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2480/3000 [08:33<01:43,  5.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2482/3000 [08:33<01:27,  5.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2485/3000 [08:34<01:19,  6.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2486/3000 [08:34<01:50,  4.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2488/3000 [08:34<01:28,  5.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2489/3000 [08:35<02:13,  3.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2492/3000 [08:35<01:34,  5.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2494/3000 [08:36<02:22,  3.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2501/3000 [08:37<01:04,  7.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▎ | 2506/3000 [08:38<01:13,  6.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▎ | 2508/3000 [08:38<01:32,  5.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▎ | 2510/3000 [08:39<01:46,  4.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▎ | 2512/3000 [08:39<01:24,  5.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2514/3000 [08:39<01:33,  5.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2516/3000 [08:40<01:40,  4.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2517/3000 [08:40<01:33,  5.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2520/3000 [08:41<01:32,  5.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2522/3000 [08:41<01:24,  5.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2524/3000 [08:41<01:15,  6.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2525/3000 [08:42<01:28,  5.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2527/3000 [08:42<01:14,  6.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2528/3000 [08:42<01:24,  5.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2531/3000 [08:42<01:20,  5.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2532/3000 [08:43<01:13,  6.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2534/3000 [08:43<01:37,  4.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2536/3000 [08:44<01:24,  5.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2537/3000 [08:44<01:28,  5.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▍ | 2540/3000 [08:44<01:07,  6.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▍ | 2543/3000 [08:45<01:56,  3.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2546/3000 [08:45<01:10,  6.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2548/3000 [08:46<01:03,  7.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2550/3000 [08:46<01:12,  6.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▌ | 2552/3000 [08:47<01:50,  4.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2555/3000 [08:47<01:06,  6.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2557/3000 [08:48<01:21,  5.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▌ | 2561/3000 [08:48<01:17,  5.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2567/3000 [08:49<00:56,  7.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2569/3000 [08:50<01:42,  4.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2570/3000 [08:50<01:37,  4.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2572/3000 [08:51<01:28,  4.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2575/3000 [08:52<01:36,  4.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2576/3000 [08:52<01:39,  4.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2577/3000 [08:52<01:36,  4.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2579/3000 [08:52<01:22,  5.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2581/3000 [08:53<01:12,  5.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2583/3000 [08:53<01:12,  5.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2585/3000 [08:53<00:57,  7.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2587/3000 [08:53<01:01,  6.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▋ | 2588/3000 [08:54<02:05,  3.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▋ | 2593/3000 [08:55<01:00,  6.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▋ | 2595/3000 [08:55<00:54,  7.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2597/3000 [08:56<01:12,  5.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2599/3000 [08:56<01:10,  5.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2600/3000 [08:56<01:04,  6.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2601/3000 [08:57<01:45,  3.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2603/3000 [08:57<01:49,  3.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2604/3000 [08:57<01:33,  4.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2606/3000 [08:58<01:15,  5.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2608/3000 [08:58<01:26,  4.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2612/3000 [08:59<00:58,  6.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2614/3000 [08:59<00:48,  7.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2616/3000 [08:59<00:49,  7.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2617/3000 [09:00<01:13,  5.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2619/3000 [09:00<01:22,  4.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2620/3000 [09:00<01:21,  4.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2622/3000 [09:01<01:57,  3.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2626/3000 [09:01<00:58,  6.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2628/3000 [09:02<01:03,  5.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2630/3000 [09:02<01:00,  6.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2631/3000 [09:03<01:11,  5.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2635/3000 [09:03<00:50,  7.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2636/3000 [09:03<00:52,  6.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2639/3000 [09:04<00:55,  6.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2640/3000 [09:04<01:01,  5.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2641/3000 [09:05<02:14,  2.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2642/3000 [09:05<02:03,  2.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2643/3000 [09:05<02:03,  2.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2644/3000 [09:06<02:11,  2.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2648/3000 [09:06<01:01,  5.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2649/3000 [09:06<01:11,  4.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2652/3000 [09:07<00:55,  6.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2654/3000 [09:07<00:51,  6.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▊ | 2656/3000 [09:07<00:51,  6.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▊ | 2658/3000 [09:08<00:45,  7.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▊ | 2659/3000 [09:08<00:46,  7.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▊ | 2661/3000 [09:08<01:03,  5.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▊ | 2662/3000 [09:09<01:10,  4.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2663/3000 [09:09<01:34,  3.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2665/3000 [09:09<01:11,  4.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2666/3000 [09:10<01:16,  4.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2669/3000 [09:10<00:57,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2672/3000 [09:11<01:06,  4.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2676/3000 [09:11<00:47,  6.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2677/3000 [09:11<00:56,  5.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2679/3000 [09:12<01:06,  4.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2681/3000 [09:12<01:00,  5.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2682/3000 [09:13<01:16,  4.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2684/3000 [09:13<01:13,  4.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|████████▉ | 2687/3000 [09:14<00:51,  6.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|████████▉ | 2690/3000 [09:14<00:42,  7.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 2692/3000 [09:14<00:35,  8.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|████████▉ | 2694/3000 [09:15<00:51,  5.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 2696/3000 [09:15<00:38,  7.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 2698/3000 [09:16<01:20,  3.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 2699/3000 [09:16<01:27,  3.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|█████████ | 2703/3000 [09:17<00:55,  5.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|█████████ | 2705/3000 [09:17<00:49,  6.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2706/3000 [09:17<00:54,  5.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|█████████ | 2709/3000 [09:18<01:01,  4.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|█████████ | 2713/3000 [09:18<00:34,  8.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2715/3000 [09:19<01:02,  4.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2717/3000 [09:20<01:12,  3.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2719/3000 [09:20<01:04,  4.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2722/3000 [09:21<00:51,  5.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2723/3000 [09:21<00:59,  4.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2725/3000 [09:21<00:58,  4.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2730/3000 [09:22<00:39,  6.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2731/3000 [09:22<00:38,  7.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2732/3000 [09:22<00:57,  4.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2735/3000 [09:23<00:49,  5.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2737/3000 [09:23<00:51,  5.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████▏| 2741/3000 [09:24<00:37,  6.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████▏| 2742/3000 [09:24<00:37,  6.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████▏| 2743/3000 [09:24<00:40,  6.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2745/3000 [09:25<01:03,  3.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2746/3000 [09:26<01:35,  2.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2748/3000 [09:26<01:11,  3.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2752/3000 [09:27<00:41,  5.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2753/3000 [09:27<00:37,  6.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2754/3000 [09:27<00:46,  5.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2758/3000 [09:27<00:29,  8.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2760/3000 [09:28<00:28,  8.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2762/3000 [09:28<00:38,  6.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2763/3000 [09:29<00:57,  4.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2764/3000 [09:29<01:15,  3.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2765/3000 [09:30<01:09,  3.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2768/3000 [09:30<01:08,  3.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2769/3000 [09:31<01:10,  3.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▎| 2775/3000 [09:31<00:29,  7.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2777/3000 [09:32<00:29,  7.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2779/3000 [09:32<00:31,  7.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2781/3000 [09:32<00:29,  7.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2783/3000 [09:33<01:04,  3.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2784/3000 [09:34<01:02,  3.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2786/3000 [09:34<00:53,  4.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2788/3000 [09:34<00:46,  4.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2791/3000 [09:35<00:33,  6.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2793/3000 [09:35<00:28,  7.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2795/3000 [09:36<00:34,  5.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2798/3000 [09:36<00:36,  5.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2799/3000 [09:36<00:34,  5.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2801/3000 [09:37<00:35,  5.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▎| 2805/3000 [09:37<00:31,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▎| 2806/3000 [09:38<00:32,  5.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▎| 2809/3000 [09:38<00:29,  6.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▎| 2811/3000 [09:39<00:40,  4.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2813/3000 [09:39<00:40,  4.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2815/3000 [09:40<00:49,  3.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2817/3000 [09:40<00:46,  3.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2819/3000 [09:40<00:36,  4.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2821/3000 [09:41<00:31,  5.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2822/3000 [09:41<00:37,  4.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2824/3000 [09:41<00:32,  5.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2826/3000 [09:42<00:27,  6.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2828/3000 [09:42<00:26,  6.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2829/3000 [09:42<00:40,  4.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2830/3000 [09:43<00:56,  2.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2832/3000 [09:44<00:51,  3.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2833/3000 [09:44<00:47,  3.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 2836/3000 [09:44<00:29,  5.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▍| 2839/3000 [09:45<00:25,  6.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 2841/3000 [09:45<00:20,  7.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▍| 2843/3000 [09:45<00:27,  5.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 2847/3000 [09:46<00:27,  5.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 2848/3000 [09:46<00:28,  5.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2850/3000 [09:47<00:28,  5.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2851/3000 [09:47<00:32,  4.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2852/3000 [09:48<01:00,  2.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▌| 2856/3000 [09:49<00:32,  4.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2858/3000 [09:49<00:38,  3.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▌| 2862/3000 [09:50<00:22,  6.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2864/3000 [09:50<00:22,  6.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2867/3000 [09:50<00:20,  6.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2868/3000 [09:51<00:19,  6.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2869/3000 [09:51<00:36,  3.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2872/3000 [09:52<00:36,  3.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2876/3000 [09:53<00:20,  6.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2878/3000 [09:53<00:19,  6.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2879/3000 [09:53<00:20,  5.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2882/3000 [09:54<00:21,  5.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2884/3000 [09:55<00:29,  3.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2886/3000 [09:55<00:28,  3.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▋| 2888/3000 [09:55<00:22,  5.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▋| 2889/3000 [09:56<00:28,  3.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▋| 2891/3000 [09:56<00:25,  4.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▋| 2894/3000 [09:57<00:18,  5.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2896/3000 [09:57<00:13,  7.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2898/3000 [09:57<00:10,  9.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2901/3000 [09:57<00:13,  7.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2902/3000 [09:58<00:14,  6.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2904/3000 [09:58<00:22,  4.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2905/3000 [09:59<00:24,  3.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2907/3000 [09:59<00:20,  4.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2909/3000 [10:00<00:22,  3.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2910/3000 [10:00<00:24,  3.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2911/3000 [10:00<00:25,  3.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2916/3000 [10:01<00:12,  6.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2919/3000 [10:01<00:11,  7.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2920/3000 [10:02<00:13,  6.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2922/3000 [10:02<00:12,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2925/3000 [10:02<00:13,  5.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2926/3000 [10:03<00:14,  5.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2928/3000 [10:03<00:17,  4.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2930/3000 [10:04<00:14,  4.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2933/3000 [10:04<00:12,  5.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2934/3000 [10:04<00:12,  5.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2935/3000 [10:05<00:13,  4.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2936/3000 [10:05<00:13,  4.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2938/3000 [10:05<00:10,  5.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2943/3000 [10:06<00:06,  9.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2950/3000 [10:07<00:09,  5.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2952/3000 [10:08<00:09,  5.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2953/3000 [10:08<00:10,  4.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2955/3000 [10:08<00:09,  4.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▊| 2956/3000 [10:09<00:09,  4.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▊| 2959/3000 [10:09<00:06,  6.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2964/3000 [10:09<00:03,  9.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2966/3000 [10:10<00:05,  6.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2967/3000 [10:11<00:08,  3.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2970/3000 [10:11<00:07,  4.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2971/3000 [10:12<00:06,  4.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2974/3000 [10:12<00:04,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2977/3000 [10:12<00:03,  7.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2980/3000 [10:13<00:02,  7.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2984/3000 [10:13<00:02,  6.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|█████████▉| 2986/3000 [10:14<00:02,  5.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2988/3000 [10:14<00:02,  5.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2989/3000 [10:15<00:02,  5.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2991/3000 [10:15<00:01,  6.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2992/3000 [10:15<00:02,  3.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|█████████▉| 2996/3000 [10:16<00:00,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2999/3000 [10:16<00:00,  7.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|██████████| 3000/3000 [10:17<00:00,  4.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 08:14:53.834\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m<module>\u001b[0m:\u001b[36m3\u001b[0m - \u001b[1mEvaluation metrics (after optimization): {'f1': 0.001, 'em': 0.001, 'acc': 0.52}\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "with suppress_logger_info():\n",
    "    results = textgrad_optimizer.evaluate(dataset=benchmark, eval_mode=\"test\")\n",
    "logger.info(f\"Evaluation metrics (after optimization): {results}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "fd546a1e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'f1': 0.001, 'em': 0.001, 'acc': 0.52}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "78d5904e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# len(benchmark._train_data)\n",
    "import numpy as np\n",
    "np.random.seed(2024)\n",
    "out = np.random.choice(benchmark._train_data, size=150, replace=False)\n",
    "benchmark._train_data = out\n",
    "benchmark._dev_data = out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "d686ee20",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 08:24:02.443\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m<module>\u001b[0m:\u001b[36m1\u001b[0m - \u001b[1mEvaluating workflow on test set...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   0%|          | 1/3000 [00:02<2:12:06,  2.64s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   0%|          | 4/3000 [00:03<25:39,  1.95it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   0%|          | 6/3000 [00:03<15:00,  3.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   0%|          | 9/3000 [00:03<11:09,  4.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   0%|          | 12/3000 [00:03<08:20,  5.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 16/3000 [00:04<05:52,  8.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 18/3000 [00:04<06:56,  7.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 20/3000 [00:04<06:44,  7.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 21/3000 [00:05<14:24,  3.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 22/3000 [00:06<14:23,  3.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 24/3000 [00:06<14:56,  3.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|          | 27/3000 [00:07<10:43,  4.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 30/3000 [00:07<07:19,  6.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 32/3000 [00:07<06:28,  7.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 35/3000 [00:07<05:13,  9.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 37/3000 [00:09<12:36,  3.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|▏         | 38/3000 [00:09<12:39,  3.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|▏         | 40/3000 [00:09<10:43,  4.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|▏         | 41/3000 [00:10<11:39,  4.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|▏         | 42/3000 [00:10<13:19,  3.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 45/3000 [00:10<09:26,  5.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 47/3000 [00:11<08:48,  5.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 50/3000 [00:11<08:18,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 52/3000 [00:11<07:39,  6.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 53/3000 [00:12<11:10,  4.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 56/3000 [00:12<08:14,  5.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 58/3000 [00:13<10:59,  4.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 62/3000 [00:13<07:59,  6.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 63/3000 [00:13<08:31,  5.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 65/3000 [00:14<10:21,  4.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 68/3000 [00:14<06:57,  7.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 69/3000 [00:15<08:23,  5.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 72/3000 [00:15<06:19,  7.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 73/3000 [00:16<12:10,  4.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 76/3000 [00:16<08:46,  5.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 78/3000 [00:16<08:18,  5.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 80/3000 [00:17<10:05,  4.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 81/3000 [00:17<09:41,  5.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 84/3000 [00:18<08:19,  5.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 85/3000 [00:18<12:43,  3.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 87/3000 [00:18<10:50,  4.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 88/3000 [00:19<11:36,  4.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 90/3000 [00:19<09:34,  5.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 91/3000 [00:19<09:38,  5.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 94/3000 [00:20<11:51,  4.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 95/3000 [00:20<12:27,  3.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 96/3000 [00:21<12:05,  4.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 97/3000 [00:21<12:51,  3.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 99/3000 [00:21<10:53,  4.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 102/3000 [00:22<10:28,  4.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 103/3000 [00:22<11:11,  4.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 104/3000 [00:23<13:14,  3.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▎         | 106/3000 [00:23<10:11,  4.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▎         | 107/3000 [00:23<11:15,  4.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▎         | 111/3000 [00:24<08:42,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 113/3000 [00:24<08:52,  5.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 115/3000 [00:25<08:28,  5.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 120/3000 [00:25<08:05,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 121/3000 [00:26<10:19,  4.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 123/3000 [00:26<10:23,  4.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 124/3000 [00:27<09:20,  5.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 127/3000 [00:27<08:16,  5.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 129/3000 [00:27<06:29,  7.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 132/3000 [00:28<07:26,  6.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 133/3000 [00:28<08:04,  5.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 135/3000 [00:28<08:21,  5.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 136/3000 [00:29<07:57,  6.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▍         | 138/3000 [00:29<07:29,  6.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 140/3000 [00:29<06:08,  7.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▍         | 143/3000 [00:30<13:16,  3.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▍         | 145/3000 [00:31<14:50,  3.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 147/3000 [00:31<09:46,  4.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▌         | 151/3000 [00:32<06:14,  7.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 153/3000 [00:32<06:25,  7.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 155/3000 [00:32<07:30,  6.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 157/3000 [00:33<06:53,  6.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▌         | 159/3000 [00:33<08:18,  5.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 161/3000 [00:34<09:58,  4.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▌         | 163/3000 [00:35<13:44,  3.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 166/3000 [00:36<13:32,  3.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 167/3000 [00:36<11:45,  4.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 170/3000 [00:36<08:03,  5.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 172/3000 [00:36<09:11,  5.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 174/3000 [00:37<08:42,  5.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 175/3000 [00:37<08:58,  5.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 176/3000 [00:37<10:19,  4.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 179/3000 [00:38<10:17,  4.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 181/3000 [00:39<11:12,  4.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 184/3000 [00:39<08:56,  5.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 185/3000 [00:39<09:01,  5.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▋         | 188/3000 [00:40<10:29,  4.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▋         | 190/3000 [00:40<10:16,  4.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▋         | 192/3000 [00:41<09:10,  5.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▋         | 194/3000 [00:41<07:09,  6.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 197/3000 [00:42<08:07,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 199/3000 [00:42<07:52,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 200/3000 [00:42<09:11,  5.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 202/3000 [00:43<09:33,  4.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 205/3000 [00:43<06:51,  6.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 206/3000 [00:43<06:37,  7.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 207/3000 [00:44<11:12,  4.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 208/3000 [00:44<11:06,  4.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 210/3000 [00:44<10:03,  4.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 211/3000 [00:45<11:55,  3.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 218/3000 [00:46<06:21,  7.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 221/3000 [00:47<10:38,  4.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 223/3000 [00:47<09:40,  4.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 226/3000 [00:47<08:18,  5.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 228/3000 [00:48<10:06,  4.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 229/3000 [00:48<10:52,  4.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 230/3000 [00:49<11:27,  4.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 231/3000 [00:49<13:30,  3.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 236/3000 [00:50<07:41,  5.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 238/3000 [00:50<08:23,  5.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 239/3000 [00:50<08:26,  5.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 241/3000 [00:51<09:22,  4.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 243/3000 [00:51<08:07,  5.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 246/3000 [00:51<07:03,  6.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 247/3000 [00:52<07:21,  6.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 251/3000 [00:53<11:16,  4.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 252/3000 [00:53<13:45,  3.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 254/3000 [00:54<11:29,  3.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▊         | 257/3000 [00:54<07:04,  6.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▊         | 258/3000 [00:54<07:14,  6.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▊         | 261/3000 [00:55<07:59,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 264/3000 [00:55<05:42,  7.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 265/3000 [00:56<18:45,  2.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 268/3000 [00:57<11:57,  3.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 269/3000 [00:57<12:39,  3.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 272/3000 [00:57<08:19,  5.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 274/3000 [00:58<08:22,  5.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 277/3000 [00:58<06:00,  7.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 279/3000 [00:58<05:33,  8.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 281/3000 [00:59<06:21,  7.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 282/3000 [00:59<09:37,  4.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 284/3000 [00:59<08:35,  5.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 285/3000 [01:00<12:48,  3.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|▉         | 288/3000 [01:01<09:09,  4.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 289/3000 [01:01<12:03,  3.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|▉         | 291/3000 [01:02<13:40,  3.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|▉         | 293/3000 [01:02<09:50,  4.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 294/3000 [01:02<08:37,  5.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|▉         | 298/3000 [01:03<07:22,  6.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 302/3000 [01:03<04:26, 10.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 304/3000 [01:04<09:56,  4.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 306/3000 [01:04<09:08,  4.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 307/3000 [01:05<10:31,  4.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 308/3000 [01:05<11:00,  4.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 309/3000 [01:05<11:26,  3.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 311/3000 [01:06<13:56,  3.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 315/3000 [01:07<08:50,  5.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 318/3000 [01:07<08:13,  5.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 320/3000 [01:07<07:00,  6.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 323/3000 [01:08<05:43,  7.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 325/3000 [01:08<07:04,  6.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 326/3000 [01:08<07:55,  5.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 327/3000 [01:09<08:58,  4.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 328/3000 [01:09<09:43,  4.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 329/3000 [01:09<10:38,  4.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 330/3000 [01:10<11:52,  3.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 331/3000 [01:10<12:45,  3.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 332/3000 [01:10<12:35,  3.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 334/3000 [01:10<09:18,  4.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 335/3000 [01:11<11:44,  3.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█▏        | 340/3000 [01:11<06:33,  6.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█▏        | 343/3000 [01:12<06:57,  6.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 345/3000 [01:12<07:27,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 346/3000 [01:13<08:46,  5.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 347/3000 [01:13<11:44,  3.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 349/3000 [01:13<09:24,  4.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 352/3000 [01:14<08:29,  5.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 353/3000 [01:14<08:34,  5.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 356/3000 [01:15<09:32,  4.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 359/3000 [01:15<06:07,  7.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 361/3000 [01:15<05:38,  7.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 363/3000 [01:16<10:00,  4.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 365/3000 [01:17<10:01,  4.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 369/3000 [01:18<08:27,  5.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 373/3000 [01:18<07:05,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▎        | 375/3000 [01:19<07:56,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 379/3000 [01:19<06:00,  7.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 381/3000 [01:20<07:10,  6.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 382/3000 [01:20<08:38,  5.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 385/3000 [01:21<08:19,  5.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 386/3000 [01:21<10:51,  4.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 390/3000 [01:22<09:33,  4.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 393/3000 [01:22<08:20,  5.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 396/3000 [01:23<06:21,  6.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 397/3000 [01:23<05:59,  7.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 401/3000 [01:24<06:37,  6.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 403/3000 [01:24<06:13,  6.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▎        | 405/3000 [01:25<10:46,  4.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▎        | 409/3000 [01:25<08:22,  5.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▎        | 410/3000 [01:26<12:08,  3.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▎        | 411/3000 [01:26<14:15,  3.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 413/3000 [01:27<11:29,  3.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 416/3000 [01:27<08:37,  4.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 418/3000 [01:28<08:59,  4.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 420/3000 [01:28<07:08,  6.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 421/3000 [01:28<07:19,  5.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 425/3000 [01:28<05:36,  7.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 427/3000 [01:29<09:56,  4.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 429/3000 [01:30<08:10,  5.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 430/3000 [01:30<11:44,  3.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 434/3000 [01:31<08:55,  4.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 435/3000 [01:31<08:10,  5.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▍        | 438/3000 [01:31<07:04,  6.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 441/3000 [01:32<06:23,  6.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▍        | 445/3000 [01:32<04:55,  8.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 447/3000 [01:33<07:32,  5.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 450/3000 [01:34<11:47,  3.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 452/3000 [01:35<12:02,  3.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 453/3000 [01:35<11:03,  3.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 454/3000 [01:35<11:29,  3.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 456/3000 [01:36<10:28,  4.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 457/3000 [01:36<12:19,  3.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 459/3000 [01:36<10:13,  4.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 464/3000 [01:37<05:12,  8.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 466/3000 [01:37<06:26,  6.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 468/3000 [01:37<06:05,  6.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 470/3000 [01:38<10:43,  3.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 471/3000 [01:39<16:15,  2.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 472/3000 [01:40<16:17,  2.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 473/3000 [01:40<16:17,  2.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 475/3000 [01:41<15:00,  2.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 477/3000 [01:41<11:41,  3.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 482/3000 [01:42<06:56,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 483/3000 [01:42<07:07,  5.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 485/3000 [01:42<07:56,  5.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 486/3000 [01:43<09:14,  4.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 487/3000 [01:43<09:45,  4.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▋        | 488/3000 [01:43<11:22,  3.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▋        | 489/3000 [01:44<13:21,  3.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▋        | 492/3000 [01:44<09:13,  4.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▋        | 493/3000 [01:45<13:29,  3.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▋        | 494/3000 [01:45<13:33,  3.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 496/3000 [01:45<10:56,  3.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 498/3000 [01:46<08:03,  5.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 499/3000 [01:46<09:25,  4.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 500/3000 [01:46<10:22,  4.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 501/3000 [01:47<12:56,  3.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 505/3000 [01:47<07:24,  5.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 506/3000 [01:48<08:04,  5.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 508/3000 [01:48<07:09,  5.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 510/3000 [01:48<06:20,  6.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 512/3000 [01:49<09:05,  4.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 514/3000 [01:49<08:08,  5.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 515/3000 [01:50<13:30,  3.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 518/3000 [01:51<11:00,  3.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 521/3000 [01:51<06:41,  6.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 525/3000 [01:51<04:50,  8.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 527/3000 [01:52<08:06,  5.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 530/3000 [01:52<07:05,  5.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 531/3000 [01:53<11:09,  3.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 533/3000 [01:54<11:41,  3.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 535/3000 [01:54<10:19,  3.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 539/3000 [01:55<06:51,  5.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 540/3000 [01:55<06:51,  5.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 544/3000 [01:55<04:53,  8.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 546/3000 [01:55<04:46,  8.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 548/3000 [01:56<05:46,  7.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 550/3000 [01:57<10:41,  3.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 554/3000 [01:58<09:47,  4.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▊        | 556/3000 [01:58<09:47,  4.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▊        | 557/3000 [01:59<11:42,  3.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▊        | 558/3000 [01:59<13:57,  2.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▊        | 559/3000 [02:00<15:16,  2.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▊        | 561/3000 [02:00<10:23,  3.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 563/3000 [02:00<08:41,  4.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 564/3000 [02:00<08:59,  4.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 568/3000 [02:02<10:55,  3.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 570/3000 [02:02<09:06,  4.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 572/3000 [02:03<10:44,  3.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 573/3000 [02:03<11:24,  3.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 576/3000 [02:04<10:00,  4.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 578/3000 [02:04<07:27,  5.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 580/3000 [02:04<07:05,  5.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|█▉        | 585/3000 [02:05<06:12,  6.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 587/3000 [02:06<07:40,  5.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 589/3000 [02:07<10:48,  3.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 591/3000 [02:07<08:51,  4.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 592/3000 [02:07<09:32,  4.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 593/3000 [02:08<11:17,  3.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|█▉        | 597/3000 [02:08<07:15,  5.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 599/3000 [02:08<05:28,  7.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 601/3000 [02:09<08:09,  4.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|██        | 603/3000 [02:09<09:12,  4.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 604/3000 [02:10<08:42,  4.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 605/3000 [02:10<11:58,  3.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|██        | 607/3000 [02:11<10:20,  3.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 610/3000 [02:11<05:32,  7.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 612/3000 [02:11<05:14,  7.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 614/3000 [02:12<08:30,  4.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 615/3000 [02:12<09:30,  4.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 618/3000 [02:12<06:59,  5.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 620/3000 [02:13<05:58,  6.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 621/3000 [02:13<07:19,  5.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 622/3000 [02:13<08:44,  4.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 623/3000 [02:14<10:50,  3.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 625/3000 [02:14<08:26,  4.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 627/3000 [02:14<08:40,  4.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 630/3000 [02:15<06:18,  6.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 634/3000 [02:15<04:42,  8.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 636/3000 [02:15<04:51,  8.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██▏       | 638/3000 [02:16<09:22,  4.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██▏       | 640/3000 [02:17<07:29,  5.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██▏       | 642/3000 [02:17<07:33,  5.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██▏       | 643/3000 [02:17<09:05,  4.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 645/3000 [02:18<08:54,  4.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 646/3000 [02:18<09:32,  4.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 648/3000 [02:19<10:06,  3.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 650/3000 [02:19<09:00,  4.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 652/3000 [02:19<07:49,  5.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 654/3000 [02:20<07:35,  5.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 655/3000 [02:20<07:27,  5.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 657/3000 [02:20<07:18,  5.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 658/3000 [02:21<08:08,  4.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 660/3000 [02:21<09:49,  3.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 662/3000 [02:22<10:13,  3.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 666/3000 [02:22<06:33,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 667/3000 [02:22<06:18,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 669/3000 [02:23<07:01,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 671/3000 [02:23<05:16,  7.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 674/3000 [02:24<05:32,  6.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 677/3000 [02:24<06:36,  5.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 680/3000 [02:24<04:45,  8.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 681/3000 [02:25<07:28,  5.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 682/3000 [02:25<07:59,  4.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 683/3000 [02:25<08:06,  4.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 684/3000 [02:26<10:42,  3.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 686/3000 [02:26<09:27,  4.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 692/3000 [02:27<04:59,  7.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 693/3000 [02:27<05:58,  6.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 696/3000 [02:27<05:04,  7.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 698/3000 [02:28<08:38,  4.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 700/3000 [02:28<07:15,  5.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 702/3000 [02:29<06:22,  6.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 703/3000 [02:29<09:16,  4.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▎       | 705/3000 [02:30<09:52,  3.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▎       | 707/3000 [02:30<08:06,  4.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▎       | 709/3000 [02:31<08:17,  4.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▎       | 712/3000 [02:31<06:27,  5.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 717/3000 [02:32<06:06,  6.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 719/3000 [02:32<06:00,  6.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 720/3000 [02:32<06:42,  5.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 721/3000 [02:33<08:27,  4.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 725/3000 [02:33<05:58,  6.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 728/3000 [02:34<05:59,  6.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 730/3000 [02:34<07:09,  5.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 731/3000 [02:35<08:43,  4.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 732/3000 [02:35<12:04,  3.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▍       | 736/3000 [02:36<06:58,  5.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 738/3000 [02:36<05:41,  6.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▍       | 742/3000 [02:36<04:16,  8.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 744/3000 [02:37<07:40,  4.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 746/3000 [02:37<08:10,  4.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 748/3000 [02:38<09:06,  4.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 749/3000 [02:38<09:05,  4.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 750/3000 [02:39<10:37,  3.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 752/3000 [02:39<09:37,  3.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 754/3000 [02:39<07:55,  4.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 755/3000 [02:40<08:53,  4.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 757/3000 [02:40<09:24,  3.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 759/3000 [02:41<08:02,  4.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 762/3000 [02:41<06:38,  5.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 766/3000 [02:41<05:54,  6.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 767/3000 [02:42<05:31,  6.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 769/3000 [02:42<09:09,  4.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 772/3000 [02:43<10:02,  3.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 775/3000 [02:44<07:16,  5.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 776/3000 [02:44<07:10,  5.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 777/3000 [02:44<07:32,  4.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 780/3000 [02:45<07:33,  4.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 782/3000 [02:45<05:50,  6.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 785/3000 [02:46<06:04,  6.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 786/3000 [02:46<05:58,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▋       | 789/3000 [02:46<05:28,  6.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▋       | 791/3000 [02:47<06:52,  5.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▋       | 792/3000 [02:47<07:35,  4.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▋       | 794/3000 [02:47<06:52,  5.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▋       | 795/3000 [02:48<07:32,  4.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 796/3000 [02:48<10:08,  3.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 799/3000 [02:48<07:01,  5.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 800/3000 [02:49<06:14,  5.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 803/3000 [02:49<06:34,  5.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 804/3000 [02:49<06:55,  5.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 806/3000 [02:50<08:20,  4.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 807/3000 [02:50<08:45,  4.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 810/3000 [02:51<08:06,  4.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 811/3000 [02:51<08:47,  4.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 814/3000 [02:52<06:39,  5.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 818/3000 [02:52<05:29,  6.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 819/3000 [02:53<07:34,  4.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 820/3000 [02:53<07:55,  4.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 821/3000 [02:53<08:39,  4.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 822/3000 [02:54<09:32,  3.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 825/3000 [02:54<07:58,  4.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 826/3000 [02:55<07:52,  4.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 828/3000 [02:55<08:37,  4.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 830/3000 [02:55<06:06,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 832/3000 [02:55<05:30,  6.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 835/3000 [02:56<04:20,  8.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 836/3000 [02:56<05:08,  7.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 838/3000 [02:56<05:22,  6.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 839/3000 [02:57<07:38,  4.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 841/3000 [02:57<07:08,  5.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 843/3000 [02:58<07:29,  4.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 845/3000 [02:58<06:16,  5.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 846/3000 [02:58<09:30,  3.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 849/3000 [02:59<06:53,  5.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 853/3000 [02:59<04:24,  8.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 855/3000 [03:00<05:21,  6.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▊       | 858/3000 [03:00<06:24,  5.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▊       | 859/3000 [03:00<05:45,  6.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▊       | 861/3000 [03:01<05:10,  6.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 864/3000 [03:02<07:50,  4.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 866/3000 [03:02<06:03,  5.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 867/3000 [03:02<07:06,  5.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 869/3000 [03:02<06:14,  5.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 871/3000 [03:03<06:57,  5.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 874/3000 [03:04<06:24,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 875/3000 [03:04<07:18,  4.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 877/3000 [03:04<06:48,  5.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 879/3000 [03:05<07:31,  4.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 882/3000 [03:05<05:30,  6.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 883/3000 [03:06<09:13,  3.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 884/3000 [03:06<09:11,  3.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 885/3000 [03:06<08:49,  4.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|██▉       | 887/3000 [03:07<08:52,  3.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 889/3000 [03:07<07:27,  4.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 890/3000 [03:07<07:26,  4.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 892/3000 [03:07<05:53,  5.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|██▉       | 895/3000 [03:08<05:40,  6.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|██▉       | 898/3000 [03:08<05:06,  6.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|███       | 902/3000 [03:09<04:55,  7.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 903/3000 [03:10<09:28,  3.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 904/3000 [03:10<10:11,  3.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 906/3000 [03:10<08:23,  4.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|███       | 910/3000 [03:11<06:05,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|███       | 913/3000 [03:11<05:31,  6.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 914/3000 [03:12<06:28,  5.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 915/3000 [03:12<07:43,  4.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 916/3000 [03:12<08:36,  4.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 918/3000 [03:12<06:33,  5.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 921/3000 [03:13<04:53,  7.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 923/3000 [03:14<08:08,  4.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 926/3000 [03:14<06:26,  5.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 927/3000 [03:14<06:17,  5.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 931/3000 [03:15<04:28,  7.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 933/3000 [03:15<07:02,  4.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 934/3000 [03:16<08:52,  3.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 935/3000 [03:16<08:42,  3.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███▏      | 939/3000 [03:17<04:49,  7.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███▏      | 941/3000 [03:17<06:10,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███▏      | 943/3000 [03:17<05:25,  6.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███▏      | 944/3000 [03:17<05:51,  5.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 947/3000 [03:18<04:49,  7.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 949/3000 [03:18<04:01,  8.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 950/3000 [03:18<06:57,  4.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 951/3000 [03:19<07:14,  4.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 953/3000 [03:20<09:12,  3.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 954/3000 [03:20<10:05,  3.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 957/3000 [03:20<07:08,  4.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 958/3000 [03:20<06:37,  5.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 960/3000 [03:21<06:59,  4.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 964/3000 [03:21<04:53,  6.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 966/3000 [03:22<05:47,  5.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 968/3000 [03:22<05:18,  6.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 970/3000 [03:23<06:42,  5.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 971/3000 [03:23<07:42,  4.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 972/3000 [03:23<08:08,  4.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▎      | 975/3000 [03:24<06:06,  5.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 977/3000 [03:24<05:35,  6.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 979/3000 [03:24<05:47,  5.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 982/3000 [03:24<04:08,  8.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 983/3000 [03:25<04:05,  8.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 985/3000 [03:25<04:11,  8.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 986/3000 [03:25<06:19,  5.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 989/3000 [03:26<04:53,  6.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 990/3000 [03:26<10:37,  3.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 993/3000 [03:27<06:50,  4.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 994/3000 [03:27<07:24,  4.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 998/3000 [03:28<05:01,  6.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 1001/3000 [03:28<05:57,  5.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 1002/3000 [03:29<06:37,  5.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 1004/3000 [03:29<06:24,  5.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▎      | 1009/3000 [03:29<03:29,  9.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▎      | 1011/3000 [03:30<05:39,  5.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▎      | 1012/3000 [03:31<08:08,  4.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 1015/3000 [03:31<07:27,  4.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 1022/3000 [03:32<05:05,  6.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1024/3000 [03:32<04:30,  7.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1026/3000 [03:33<05:20,  6.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1028/3000 [03:33<05:03,  6.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1030/3000 [03:33<04:58,  6.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 1033/3000 [03:34<05:39,  5.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1034/3000 [03:34<05:45,  5.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▍      | 1037/3000 [03:35<06:59,  4.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▍      | 1039/3000 [03:36<06:33,  4.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▍      | 1042/3000 [03:36<06:27,  5.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 1045/3000 [03:36<04:40,  6.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 1046/3000 [03:37<04:43,  6.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▌      | 1050/3000 [03:37<04:45,  6.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▌      | 1052/3000 [03:38<07:29,  4.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 1054/3000 [03:38<05:42,  5.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▌      | 1057/3000 [03:39<06:55,  4.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▌      | 1059/3000 [03:39<05:59,  5.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▌      | 1061/3000 [03:40<06:00,  5.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▌      | 1064/3000 [03:41<06:39,  4.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 1068/3000 [03:41<03:52,  8.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 1072/3000 [03:42<04:35,  7.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1074/3000 [03:42<06:44,  4.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 1077/3000 [03:43<05:11,  6.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 1081/3000 [03:43<05:20,  5.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 1084/3000 [03:44<07:38,  4.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▋      | 1090/3000 [03:45<04:51,  6.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▋      | 1092/3000 [03:45<04:52,  6.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▋      | 1095/3000 [03:46<04:45,  6.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1096/3000 [03:46<05:16,  6.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1097/3000 [03:47<07:05,  4.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1099/3000 [03:47<06:20,  5.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1100/3000 [03:47<06:36,  4.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 1102/3000 [03:48<07:25,  4.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1103/3000 [03:48<06:29,  4.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 1105/3000 [03:49<07:41,  4.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 1107/3000 [03:49<05:44,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 1111/3000 [03:50<07:08,  4.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1114/3000 [03:50<05:14,  6.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 1116/3000 [03:50<05:13,  6.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1117/3000 [03:51<05:27,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1119/3000 [03:51<08:09,  3.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 1122/3000 [03:52<06:26,  4.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1127/3000 [03:53<04:14,  7.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1130/3000 [03:53<04:13,  7.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1133/3000 [03:53<03:38,  8.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1134/3000 [03:53<04:12,  7.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1136/3000 [03:54<08:24,  3.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1137/3000 [03:55<08:57,  3.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1140/3000 [03:56<07:25,  4.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1141/3000 [03:56<06:28,  4.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1143/3000 [03:56<05:24,  5.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1146/3000 [03:56<04:19,  7.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1148/3000 [03:57<04:06,  7.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1150/3000 [03:57<04:04,  7.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1151/3000 [03:57<07:14,  4.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1152/3000 [03:58<08:40,  3.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1154/3000 [03:58<08:40,  3.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▊      | 1156/3000 [03:59<07:45,  3.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▊      | 1161/3000 [04:00<06:02,  5.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 1164/3000 [04:00<05:29,  5.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1167/3000 [04:00<04:46,  6.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1168/3000 [04:01<05:48,  5.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 1170/3000 [04:01<06:50,  4.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 1172/3000 [04:02<07:31,  4.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 1175/3000 [04:02<04:57,  6.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 1178/3000 [04:03<04:27,  6.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 1181/3000 [04:04<06:12,  4.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1183/3000 [04:04<05:42,  5.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|███▉      | 1185/3000 [04:05<06:02,  5.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 1187/3000 [04:05<04:23,  6.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 1189/3000 [04:05<06:03,  4.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 1190/3000 [04:06<07:05,  4.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|███▉      | 1192/3000 [04:06<07:13,  4.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|███▉      | 1194/3000 [04:07<06:22,  4.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 1195/3000 [04:07<08:24,  3.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 1196/3000 [04:07<08:56,  3.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 1197/3000 [04:08<10:03,  2.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 1199/3000 [04:08<07:22,  4.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|████      | 1202/3000 [04:09<05:49,  5.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1205/3000 [04:09<04:01,  7.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1208/3000 [04:09<04:09,  7.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1209/3000 [04:09<05:05,  5.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|████      | 1212/3000 [04:10<04:50,  6.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|████      | 1214/3000 [04:11<06:33,  4.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1216/3000 [04:11<05:04,  5.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1218/3000 [04:11<05:26,  5.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1220/3000 [04:13<10:38,  2.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1222/3000 [04:13<06:58,  4.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1225/3000 [04:13<05:50,  5.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1228/3000 [04:14<03:59,  7.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1229/3000 [04:14<04:13,  6.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1232/3000 [04:14<03:33,  8.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1234/3000 [04:15<05:09,  5.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1236/3000 [04:15<05:32,  5.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████▏     | 1238/3000 [04:15<05:20,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████▏     | 1239/3000 [04:16<07:50,  3.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████▏     | 1240/3000 [04:16<07:23,  3.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████▏     | 1242/3000 [04:17<06:41,  4.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████▏     | 1243/3000 [04:17<06:25,  4.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1245/3000 [04:17<05:19,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1246/3000 [04:18<08:31,  3.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1248/3000 [04:18<08:09,  3.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1251/3000 [04:19<08:12,  3.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1257/3000 [04:19<03:30,  8.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1259/3000 [04:20<04:18,  6.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1261/3000 [04:20<04:51,  5.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1262/3000 [04:21<05:17,  5.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1264/3000 [04:21<05:30,  5.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1266/3000 [04:22<09:39,  2.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1268/3000 [04:23<07:08,  4.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1272/3000 [04:23<03:50,  7.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1274/3000 [04:23<03:48,  7.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1276/3000 [04:23<03:43,  7.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1279/3000 [04:24<03:39,  7.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1281/3000 [04:24<05:31,  5.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1283/3000 [04:25<06:00,  4.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1285/3000 [04:26<06:44,  4.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1288/3000 [04:26<06:45,  4.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1289/3000 [04:27<06:37,  4.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1292/3000 [04:27<05:38,  5.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1295/3000 [04:27<05:07,  5.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1298/3000 [04:28<03:51,  7.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1299/3000 [04:28<03:46,  7.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1302/3000 [04:28<03:43,  7.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1303/3000 [04:29<06:16,  4.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1304/3000 [04:29<06:27,  4.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▎     | 1306/3000 [04:29<05:54,  4.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▎     | 1309/3000 [04:30<05:17,  5.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▎     | 1311/3000 [04:30<04:56,  5.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1314/3000 [04:31<04:13,  6.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1316/3000 [04:31<04:38,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1317/3000 [04:32<07:25,  3.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1320/3000 [04:32<05:31,  5.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1322/3000 [04:33<05:08,  5.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1323/3000 [04:33<04:45,  5.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1324/3000 [04:33<08:07,  3.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1327/3000 [04:34<05:24,  5.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1329/3000 [04:34<05:10,  5.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1331/3000 [04:34<04:44,  5.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1332/3000 [04:35<06:27,  4.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1334/3000 [04:35<06:17,  4.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1338/3000 [04:36<05:28,  5.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1340/3000 [04:36<04:55,  5.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▍     | 1342/3000 [04:37<05:14,  5.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▍     | 1344/3000 [04:37<05:49,  4.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1347/3000 [04:38<05:31,  4.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▌     | 1350/3000 [04:38<05:54,  4.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▌     | 1353/3000 [04:39<05:28,  5.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1355/3000 [04:39<04:08,  6.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▌     | 1358/3000 [04:40<04:20,  6.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▌     | 1360/3000 [04:40<04:28,  6.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1361/3000 [04:40<04:06,  6.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1363/3000 [04:40<04:22,  6.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1364/3000 [04:41<07:21,  3.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1365/3000 [04:41<07:37,  3.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1367/3000 [04:42<07:19,  3.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1368/3000 [04:42<09:06,  2.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1370/3000 [04:43<06:17,  4.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1374/3000 [04:43<04:30,  6.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1376/3000 [04:44<04:31,  5.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1379/3000 [04:44<04:18,  6.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1381/3000 [04:44<04:13,  6.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1383/3000 [04:45<04:43,  5.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1385/3000 [04:45<05:56,  4.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1387/3000 [04:46<06:16,  4.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▋     | 1390/3000 [04:46<04:59,  5.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▋     | 1392/3000 [04:47<05:15,  5.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▋     | 1393/3000 [04:47<06:43,  3.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▋     | 1395/3000 [04:48<05:50,  4.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1397/3000 [04:48<05:53,  4.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1399/3000 [04:48<04:33,  5.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1401/3000 [04:48<03:58,  6.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1404/3000 [04:49<05:30,  4.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1405/3000 [04:50<06:08,  4.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1408/3000 [04:50<05:11,  5.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1410/3000 [04:51<06:26,  4.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1412/3000 [04:51<04:54,  5.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1413/3000 [04:51<05:50,  4.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1414/3000 [04:52<07:09,  3.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1415/3000 [04:52<07:36,  3.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1417/3000 [04:52<05:33,  4.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1418/3000 [04:52<05:29,  4.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1419/3000 [04:53<05:32,  4.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1421/3000 [04:53<06:44,  3.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1424/3000 [04:54<07:53,  3.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1426/3000 [04:55<06:20,  4.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1430/3000 [04:55<04:32,  5.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1432/3000 [04:55<03:39,  7.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1434/3000 [04:56<04:36,  5.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1435/3000 [04:56<04:51,  5.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1436/3000 [04:56<05:15,  4.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1437/3000 [04:57<05:56,  4.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1438/3000 [04:57<06:49,  3.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1440/3000 [04:58<06:59,  3.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1441/3000 [04:58<08:17,  3.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1443/3000 [04:58<06:04,  4.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1446/3000 [04:59<05:40,  4.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1447/3000 [04:59<05:21,  4.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1448/3000 [05:00<06:28,  3.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1451/3000 [05:00<04:08,  6.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1453/3000 [05:00<04:42,  5.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1454/3000 [05:01<05:34,  4.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▊     | 1456/3000 [05:01<06:27,  3.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▊     | 1457/3000 [05:02<07:10,  3.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▊     | 1460/3000 [05:02<05:02,  5.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1464/3000 [05:03<03:48,  6.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1466/3000 [05:03<03:35,  7.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1468/3000 [05:03<03:16,  7.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1470/3000 [05:03<03:19,  7.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1471/3000 [05:04<04:22,  5.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1473/3000 [05:04<04:49,  5.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1475/3000 [05:05<05:28,  4.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1478/3000 [05:06<08:06,  3.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1480/3000 [05:06<06:13,  4.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1484/3000 [05:07<04:32,  5.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1486/3000 [05:07<03:21,  7.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|████▉     | 1491/3000 [05:08<04:32,  5.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1493/3000 [05:08<03:58,  6.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1495/3000 [05:09<05:56,  4.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1497/3000 [05:09<05:14,  4.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1499/3000 [05:10<05:15,  4.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 1502/3000 [05:11<05:17,  4.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 1508/3000 [05:11<03:02,  8.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 1511/3000 [05:12<03:53,  6.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1512/3000 [05:12<03:39,  6.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1514/3000 [05:13<07:38,  3.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1516/3000 [05:13<06:53,  3.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1517/3000 [05:14<06:33,  3.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1518/3000 [05:14<06:25,  3.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1520/3000 [05:14<05:14,  4.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1525/3000 [05:15<02:58,  8.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1527/3000 [05:15<03:56,  6.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1529/3000 [05:16<05:13,  4.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1530/3000 [05:16<06:47,  3.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1534/3000 [05:17<04:59,  4.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1537/3000 [05:18<04:27,  5.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████▏    | 1539/3000 [05:18<03:58,  6.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████▏    | 1540/3000 [05:19<06:48,  3.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████▏    | 1543/3000 [05:19<05:26,  4.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1545/3000 [05:19<04:07,  5.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1546/3000 [05:20<04:56,  4.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1548/3000 [05:20<04:25,  5.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1550/3000 [05:20<04:17,  5.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1551/3000 [05:21<04:27,  5.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1553/3000 [05:21<03:50,  6.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1556/3000 [05:22<04:32,  5.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1557/3000 [05:22<04:04,  5.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1559/3000 [05:22<04:48,  5.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1560/3000 [05:22<05:38,  4.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1562/3000 [05:24<08:20,  2.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1567/3000 [05:24<04:19,  5.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1570/3000 [05:24<03:18,  7.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1572/3000 [05:25<03:32,  6.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1574/3000 [05:25<03:09,  7.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1577/3000 [05:25<03:57,  6.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1578/3000 [05:26<05:15,  4.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1580/3000 [05:27<06:12,  3.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1582/3000 [05:27<05:51,  4.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1584/3000 [05:27<05:36,  4.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1585/3000 [05:28<05:15,  4.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1587/3000 [05:28<05:06,  4.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1590/3000 [05:28<03:38,  6.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1591/3000 [05:29<04:14,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1594/3000 [05:29<04:07,  5.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1597/3000 [05:29<03:10,  7.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1598/3000 [05:30<04:27,  5.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1600/3000 [05:30<04:13,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1601/3000 [05:30<04:51,  4.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1604/3000 [05:32<06:18,  3.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▎    | 1608/3000 [05:32<04:10,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▎    | 1609/3000 [05:32<04:01,  5.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▎    | 1610/3000 [05:33<05:27,  4.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1615/3000 [05:33<03:05,  7.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1617/3000 [05:34<03:58,  5.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1619/3000 [05:34<04:41,  4.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1620/3000 [05:35<05:29,  4.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1621/3000 [05:35<06:07,  3.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1624/3000 [05:36<05:22,  4.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1626/3000 [05:36<04:01,  5.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1629/3000 [05:36<03:27,  6.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1631/3000 [05:36<03:16,  6.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1633/3000 [05:37<03:25,  6.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1635/3000 [05:37<03:10,  7.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▍    | 1637/3000 [05:37<03:21,  6.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▍    | 1639/3000 [05:38<05:31,  4.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1640/3000 [05:39<08:15,  2.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1641/3000 [05:39<08:05,  2.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1644/3000 [05:39<04:38,  4.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1646/3000 [05:40<03:52,  5.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1648/3000 [05:40<04:08,  5.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1652/3000 [05:40<02:41,  8.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1654/3000 [05:41<04:53,  4.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▌    | 1656/3000 [05:42<05:22,  4.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▌    | 1659/3000 [05:42<03:34,  6.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1661/3000 [05:43<05:01,  4.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▌    | 1664/3000 [05:43<04:10,  5.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1665/3000 [05:44<04:12,  5.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1668/3000 [05:44<03:16,  6.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1669/3000 [05:44<04:38,  4.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1671/3000 [05:45<04:36,  4.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1673/3000 [05:45<05:14,  4.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1677/3000 [05:46<04:03,  5.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1678/3000 [05:46<04:24,  5.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1679/3000 [05:47<05:29,  4.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1681/3000 [05:47<06:24,  3.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1683/3000 [05:48<05:07,  4.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1684/3000 [05:48<05:08,  4.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1686/3000 [05:48<04:00,  5.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1687/3000 [05:49<04:49,  4.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▋    | 1690/3000 [05:49<03:57,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▋    | 1691/3000 [05:49<04:05,  5.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▋    | 1692/3000 [05:50<04:58,  4.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▋    | 1694/3000 [05:50<04:50,  4.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▋    | 1695/3000 [05:50<05:07,  4.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1697/3000 [05:51<05:53,  3.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1699/3000 [05:51<04:07,  5.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1703/3000 [05:52<03:02,  7.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1704/3000 [05:52<03:01,  7.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1707/3000 [05:53<04:41,  4.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1709/3000 [05:53<03:43,  5.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1712/3000 [05:53<02:51,  7.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1713/3000 [05:53<03:04,  6.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1715/3000 [05:54<03:37,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1718/3000 [05:54<03:41,  5.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1720/3000 [05:55<04:01,  5.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1721/3000 [05:55<04:05,  5.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1722/3000 [05:56<06:34,  3.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1726/3000 [05:56<03:48,  5.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1728/3000 [05:56<02:49,  7.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1731/3000 [05:57<03:16,  6.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1733/3000 [05:57<04:33,  4.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1734/3000 [05:58<04:02,  5.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1735/3000 [05:58<04:15,  4.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1736/3000 [05:58<05:19,  3.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1738/3000 [05:59<05:09,  4.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1740/3000 [05:59<05:29,  3.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1741/3000 [06:00<08:24,  2.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1746/3000 [06:01<04:06,  5.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1750/3000 [06:01<02:26,  8.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1752/3000 [06:01<02:55,  7.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1754/3000 [06:02<03:43,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1755/3000 [06:03<06:25,  3.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▊    | 1756/3000 [06:03<06:44,  3.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▊    | 1757/3000 [06:04<08:17,  2.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▊    | 1759/3000 [06:04<06:20,  3.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▊    | 1760/3000 [06:04<06:55,  2.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1765/3000 [06:05<03:27,  5.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1768/3000 [06:05<03:05,  6.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1770/3000 [06:06<03:35,  5.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1772/3000 [06:06<03:31,  5.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1773/3000 [06:06<03:39,  5.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1775/3000 [06:07<04:20,  4.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1777/3000 [06:07<04:49,  4.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1779/3000 [06:08<04:15,  4.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1780/3000 [06:08<05:35,  3.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1781/3000 [06:08<05:25,  3.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|█████▉    | 1785/3000 [06:09<03:06,  6.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1789/3000 [06:10<03:19,  6.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1790/3000 [06:10<03:30,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|█████▉    | 1793/3000 [06:10<03:15,  6.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1794/3000 [06:11<04:01,  4.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|█████▉    | 1796/3000 [06:12<06:23,  3.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|█████▉    | 1798/3000 [06:12<04:55,  4.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|██████    | 1801/3000 [06:13<04:17,  4.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1805/3000 [06:13<02:07,  9.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1807/3000 [06:13<01:55, 10.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|██████    | 1811/3000 [06:14<03:42,  5.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1813/3000 [06:14<02:58,  6.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1815/3000 [06:15<04:29,  4.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1816/3000 [06:15<04:43,  4.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1818/3000 [06:16<05:21,  3.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1820/3000 [06:17<05:36,  3.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1821/3000 [06:17<05:37,  3.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1823/3000 [06:17<04:50,  4.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1826/3000 [06:18<03:50,  5.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1828/3000 [06:18<04:12,  4.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1830/3000 [06:19<03:25,  5.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1831/3000 [06:20<07:00,  2.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1833/3000 [06:20<05:33,  3.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1835/3000 [06:20<04:15,  4.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1836/3000 [06:21<04:22,  4.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1837/3000 [06:21<04:33,  4.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████▏   | 1841/3000 [06:21<02:52,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████▏   | 1842/3000 [06:22<04:31,  4.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1846/3000 [06:22<02:53,  6.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1848/3000 [06:23<04:09,  4.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1849/3000 [06:23<04:22,  4.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1851/3000 [06:24<04:26,  4.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1852/3000 [06:24<04:19,  4.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1854/3000 [06:24<03:32,  5.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1856/3000 [06:25<03:28,  5.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1857/3000 [06:25<04:18,  4.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1859/3000 [06:25<03:38,  5.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1861/3000 [06:26<04:27,  4.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1863/3000 [06:26<04:20,  4.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1865/3000 [06:27<04:44,  4.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1866/3000 [06:27<04:01,  4.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1868/3000 [06:27<03:43,  5.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1869/3000 [06:28<04:20,  4.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1871/3000 [06:28<05:03,  3.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1873/3000 [06:29<04:22,  4.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1878/3000 [06:30<03:30,  5.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1883/3000 [06:31<04:07,  4.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1886/3000 [06:32<03:22,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1889/3000 [06:32<03:34,  5.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1891/3000 [06:33<04:27,  4.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1893/3000 [06:33<03:56,  4.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1895/3000 [06:34<04:25,  4.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1896/3000 [06:34<04:36,  3.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1898/3000 [06:34<04:23,  4.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1899/3000 [06:35<04:55,  3.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1902/3000 [06:35<04:17,  4.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1903/3000 [06:36<05:36,  3.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▎   | 1906/3000 [06:37<04:12,  4.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▎   | 1907/3000 [06:37<04:02,  4.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▎   | 1910/3000 [06:37<03:34,  5.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▎   | 1911/3000 [06:37<03:26,  5.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1913/3000 [06:38<04:01,  4.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1914/3000 [06:38<04:50,  3.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1916/3000 [06:39<04:20,  4.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1918/3000 [06:39<03:58,  4.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1919/3000 [06:40<04:13,  4.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1920/3000 [06:40<05:36,  3.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1923/3000 [06:40<03:45,  4.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1924/3000 [06:41<04:13,  4.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1927/3000 [06:41<03:53,  4.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1929/3000 [06:42<03:17,  5.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1930/3000 [06:42<04:28,  3.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1932/3000 [06:43<06:03,  2.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1935/3000 [06:44<03:57,  4.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1937/3000 [06:44<02:59,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▍   | 1940/3000 [06:44<02:54,  6.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1941/3000 [06:45<03:43,  4.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▍   | 1943/3000 [06:45<03:50,  4.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1944/3000 [06:46<06:17,  2.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1946/3000 [06:47<06:46,  2.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1948/3000 [06:47<05:10,  3.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1951/3000 [06:48<04:03,  4.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1953/3000 [06:48<03:51,  4.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1954/3000 [06:49<05:22,  3.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1955/3000 [06:49<05:15,  3.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 1957/3000 [06:50<05:29,  3.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 1959/3000 [06:50<05:26,  3.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 1961/3000 [06:51<03:53,  4.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1962/3000 [06:51<03:33,  4.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1964/3000 [06:51<03:07,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1966/3000 [06:52<03:51,  4.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1967/3000 [06:52<03:21,  5.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1969/3000 [06:52<03:30,  4.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1970/3000 [06:53<04:03,  4.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1972/3000 [06:53<03:32,  4.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1973/3000 [06:54<05:31,  3.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1974/3000 [06:54<05:43,  2.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1976/3000 [06:54<04:45,  3.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1977/3000 [06:55<04:39,  3.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1979/3000 [06:55<04:55,  3.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1982/3000 [06:56<03:24,  4.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1985/3000 [06:56<02:44,  6.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1987/3000 [06:57<03:14,  5.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▋   | 1989/3000 [06:57<03:43,  4.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▋   | 1991/3000 [06:58<04:01,  4.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▋   | 1992/3000 [06:58<03:55,  4.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▋   | 1994/3000 [06:59<04:22,  3.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 1997/3000 [06:59<03:28,  4.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 1998/3000 [07:00<04:31,  3.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 1999/3000 [07:00<04:20,  3.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2000/3000 [07:00<04:26,  3.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 2002/3000 [07:01<04:40,  3.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2004/3000 [07:01<03:46,  4.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 2007/3000 [07:02<02:48,  5.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2008/3000 [07:02<03:13,  5.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2009/3000 [07:03<05:07,  3.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2011/3000 [07:03<04:05,  4.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2012/3000 [07:03<05:35,  2.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2013/3000 [07:04<05:35,  2.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2014/3000 [07:04<05:16,  3.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 2017/3000 [07:05<03:41,  4.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 2019/3000 [07:05<03:43,  4.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2020/3000 [07:05<03:59,  4.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2021/3000 [07:06<03:48,  4.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2022/3000 [07:06<05:54,  2.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2026/3000 [07:07<03:14,  5.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2027/3000 [07:07<02:55,  5.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2028/3000 [07:08<05:49,  2.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2032/3000 [07:08<03:23,  4.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2034/3000 [07:08<02:36,  6.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2037/3000 [07:09<02:28,  6.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2038/3000 [07:09<02:31,  6.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2039/3000 [07:09<03:07,  5.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2040/3000 [07:10<05:03,  3.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2041/3000 [07:10<05:32,  2.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2042/3000 [07:11<05:48,  2.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2044/3000 [07:12<06:37,  2.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2047/3000 [07:12<03:46,  4.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2050/3000 [07:13<02:54,  5.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2052/3000 [07:13<02:09,  7.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▊   | 2056/3000 [07:14<02:41,  5.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▊   | 2058/3000 [07:14<02:14,  7.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▊   | 2060/3000 [07:15<03:05,  5.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▊   | 2062/3000 [07:15<03:19,  4.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 2064/3000 [07:16<04:30,  3.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2065/3000 [07:16<04:49,  3.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 2067/3000 [07:17<04:42,  3.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2069/3000 [07:17<03:47,  4.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2070/3000 [07:18<04:14,  3.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2071/3000 [07:18<04:46,  3.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2072/3000 [07:19<04:42,  3.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2074/3000 [07:19<03:22,  4.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2075/3000 [07:19<03:21,  4.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 2079/3000 [07:20<02:55,  5.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 2081/3000 [07:20<03:40,  4.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 2083/3000 [07:21<03:06,  4.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 2085/3000 [07:21<02:27,  6.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 2086/3000 [07:21<03:49,  3.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|██████▉   | 2089/3000 [07:22<02:50,  5.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 2090/3000 [07:23<04:52,  3.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|██████▉   | 2093/3000 [07:23<04:31,  3.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 2095/3000 [07:24<03:36,  4.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|██████▉   | 2097/3000 [07:24<03:20,  4.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|██████▉   | 2099/3000 [07:25<04:31,  3.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|███████   | 2101/3000 [07:26<03:45,  3.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|███████   | 2103/3000 [07:26<03:07,  4.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 2105/3000 [07:26<03:14,  4.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 2106/3000 [07:27<04:00,  3.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 2107/3000 [07:27<04:56,  3.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 2108/3000 [07:28<05:30,  2.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 2110/3000 [07:28<04:17,  3.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 2112/3000 [07:28<03:43,  3.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 2113/3000 [07:29<04:02,  3.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 2114/3000 [07:29<04:35,  3.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 2115/3000 [07:30<04:38,  3.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 2119/3000 [07:30<03:06,  4.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2121/3000 [07:31<03:40,  3.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2123/3000 [07:31<03:19,  4.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2124/3000 [07:32<03:27,  4.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2125/3000 [07:32<04:55,  2.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2126/3000 [07:33<05:27,  2.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2127/3000 [07:34<07:16,  2.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2128/3000 [07:34<06:16,  2.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 2132/3000 [07:34<03:19,  4.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2133/3000 [07:35<04:24,  3.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 2135/3000 [07:36<04:30,  3.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 2137/3000 [07:36<03:11,  4.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████▏  | 2138/3000 [07:36<03:41,  3.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████▏  | 2141/3000 [07:36<02:22,  6.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████▏  | 2143/3000 [07:37<02:51,  5.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2145/3000 [07:39<05:56,  2.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2148/3000 [07:39<04:49,  2.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2149/3000 [07:40<04:50,  2.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2151/3000 [07:40<03:56,  3.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2155/3000 [07:41<02:35,  5.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2156/3000 [07:41<03:16,  4.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2157/3000 [07:41<03:35,  3.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2158/3000 [07:42<04:02,  3.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2162/3000 [07:42<02:53,  4.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2163/3000 [07:42<02:34,  5.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2164/3000 [07:43<02:49,  4.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2166/3000 [07:43<03:28,  4.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2167/3000 [07:44<04:48,  2.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2168/3000 [07:45<06:03,  2.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2170/3000 [07:45<04:53,  2.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2171/3000 [07:46<06:38,  2.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2172/3000 [07:46<05:52,  2.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2173/3000 [07:47<05:17,  2.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▎  | 2175/3000 [07:47<04:13,  3.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2176/3000 [07:47<03:28,  3.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2177/3000 [07:47<03:42,  3.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2178/3000 [07:48<04:18,  3.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 2181/3000 [07:48<02:46,  4.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 2183/3000 [07:49<02:49,  4.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2184/3000 [07:49<04:07,  3.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2185/3000 [07:50<04:05,  3.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 2188/3000 [07:50<03:02,  4.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2189/3000 [07:50<03:19,  4.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2192/3000 [07:51<02:34,  5.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2193/3000 [07:51<03:27,  3.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 2196/3000 [07:52<02:41,  4.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2198/3000 [07:53<03:26,  3.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2200/3000 [07:53<02:43,  4.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2201/3000 [07:53<02:59,  4.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 2203/3000 [07:54<03:15,  4.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2204/3000 [07:54<03:05,  4.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▎  | 2206/3000 [07:54<02:45,  4.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▎  | 2207/3000 [07:55<04:03,  3.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▎  | 2208/3000 [07:55<04:39,  2.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▎  | 2209/3000 [07:56<04:54,  2.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▎  | 2210/3000 [07:57<07:44,  1.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 2213/3000 [07:57<04:03,  3.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 2219/3000 [07:58<01:53,  6.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2221/3000 [07:58<01:40,  7.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2222/3000 [07:59<03:26,  3.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2223/3000 [07:59<03:20,  3.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2224/3000 [07:59<03:15,  3.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2225/3000 [08:00<03:51,  3.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2226/3000 [08:00<03:36,  3.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2228/3000 [08:00<03:17,  3.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2229/3000 [08:01<03:28,  3.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 2231/3000 [08:01<03:18,  3.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 2233/3000 [08:01<02:39,  4.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▍  | 2236/3000 [08:02<01:50,  6.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2238/3000 [08:02<01:24,  9.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2240/3000 [08:02<01:25,  8.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2242/3000 [08:04<04:21,  2.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▍  | 2244/3000 [08:04<03:39,  3.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2245/3000 [08:05<04:46,  2.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2246/3000 [08:05<04:34,  2.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2248/3000 [08:05<03:35,  3.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2249/3000 [08:06<03:50,  3.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 2252/3000 [08:06<02:24,  5.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 2253/3000 [08:06<02:49,  4.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 2255/3000 [08:07<03:50,  3.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▌  | 2257/3000 [08:08<03:35,  3.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▌  | 2259/3000 [08:08<02:57,  4.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 2260/3000 [08:08<02:41,  4.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 2261/3000 [08:09<03:18,  3.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 2263/3000 [08:09<03:08,  3.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 2265/3000 [08:10<03:48,  3.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2268/3000 [08:10<02:12,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2269/3000 [08:11<02:48,  4.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 2271/3000 [08:11<02:49,  4.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 2274/3000 [08:13<03:40,  3.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2275/3000 [08:13<03:36,  3.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 2278/3000 [08:13<02:31,  4.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2280/3000 [08:14<02:48,  4.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2282/3000 [08:14<03:00,  3.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2284/3000 [08:15<02:51,  4.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2285/3000 [08:15<03:26,  3.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▋  | 2290/3000 [08:16<01:49,  6.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▋  | 2292/3000 [08:16<02:08,  5.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▋  | 2294/3000 [08:17<03:21,  3.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 2297/3000 [08:18<02:10,  5.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 2299/3000 [08:18<01:58,  5.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2300/3000 [08:18<02:31,  4.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2301/3000 [08:19<03:36,  3.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2302/3000 [08:19<03:55,  2.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 2305/3000 [08:19<02:17,  5.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2306/3000 [08:20<02:22,  4.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2308/3000 [08:20<02:19,  4.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2310/3000 [08:20<01:56,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2311/3000 [08:21<04:12,  2.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2316/3000 [08:22<01:57,  5.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2318/3000 [08:22<01:59,  5.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2319/3000 [08:22<02:07,  5.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 2321/3000 [08:23<02:59,  3.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 2323/3000 [08:24<03:06,  3.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2325/3000 [08:24<02:12,  5.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2326/3000 [08:24<02:37,  4.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2328/3000 [08:25<02:11,  5.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2329/3000 [08:25<02:17,  4.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2330/3000 [08:25<02:28,  4.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2332/3000 [08:26<03:31,  3.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2333/3000 [08:26<02:57,  3.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2336/3000 [08:27<02:05,  5.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2337/3000 [08:27<02:32,  4.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2338/3000 [08:28<03:42,  2.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2342/3000 [08:28<01:54,  5.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2344/3000 [08:28<01:39,  6.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2347/3000 [08:29<02:11,  4.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2348/3000 [08:29<02:16,  4.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2351/3000 [08:30<01:34,  6.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2354/3000 [08:30<01:19,  8.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▊  | 2356/3000 [08:30<01:48,  5.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▊  | 2357/3000 [08:31<02:09,  4.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▊  | 2359/3000 [08:32<02:56,  3.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▊  | 2360/3000 [08:32<03:36,  2.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▊  | 2361/3000 [08:32<03:17,  3.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2363/3000 [08:33<02:44,  3.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 2366/3000 [08:33<02:02,  5.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2368/3000 [08:34<02:10,  4.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 2370/3000 [08:34<02:09,  4.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2372/3000 [08:34<01:33,  6.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 2374/3000 [08:35<02:44,  3.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2375/3000 [08:35<02:22,  4.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 2378/3000 [08:36<01:39,  6.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2380/3000 [08:36<01:23,  7.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2382/3000 [08:36<01:35,  6.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2383/3000 [08:37<02:33,  4.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 2385/3000 [08:37<02:28,  4.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|███████▉  | 2387/3000 [08:38<02:36,  3.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|███████▉  | 2389/3000 [08:38<02:18,  4.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 2390/3000 [08:39<02:20,  4.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|███████▉  | 2394/3000 [08:39<02:04,  4.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 2395/3000 [08:40<03:16,  3.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 2399/3000 [08:40<01:49,  5.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 2404/3000 [08:42<02:17,  4.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2405/3000 [08:42<02:07,  4.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2407/3000 [08:43<02:43,  3.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 2409/3000 [08:43<02:30,  3.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2412/3000 [08:43<01:31,  6.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2416/3000 [08:44<01:11,  8.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2418/3000 [08:45<02:14,  4.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2422/3000 [08:45<01:37,  5.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2424/3000 [08:46<01:41,  5.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2426/3000 [08:47<02:53,  3.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2428/3000 [08:48<02:42,  3.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2429/3000 [08:48<02:32,  3.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2430/3000 [08:48<02:29,  3.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2436/3000 [08:49<01:20,  6.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2437/3000 [08:49<01:39,  5.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████▏ | 2439/3000 [08:49<01:46,  5.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████▏ | 2440/3000 [08:50<02:02,  4.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████▏ | 2442/3000 [08:50<02:21,  3.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████▏ | 2443/3000 [08:51<02:18,  4.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████▏ | 2444/3000 [08:51<02:30,  3.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2445/3000 [08:52<03:15,  2.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2449/3000 [08:52<01:46,  5.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2452/3000 [08:53<01:46,  5.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2458/3000 [08:53<01:10,  7.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2460/3000 [08:53<01:09,  7.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2462/3000 [08:54<01:58,  4.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2463/3000 [08:55<02:11,  4.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2466/3000 [08:56<02:23,  3.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2468/3000 [08:56<02:01,  4.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2469/3000 [08:56<02:04,  4.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2471/3000 [08:57<02:49,  3.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2473/3000 [08:58<02:02,  4.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▎ | 2475/3000 [08:58<01:51,  4.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2476/3000 [08:59<02:36,  3.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2477/3000 [08:59<03:05,  2.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2480/3000 [09:00<02:40,  3.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2483/3000 [09:00<01:51,  4.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2484/3000 [09:01<01:53,  4.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2486/3000 [09:01<01:55,  4.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2487/3000 [09:01<01:52,  4.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2488/3000 [09:02<03:37,  2.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2490/3000 [09:03<02:47,  3.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2492/3000 [09:03<02:08,  3.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2493/3000 [09:03<02:19,  3.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2496/3000 [09:04<01:39,  5.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2497/3000 [09:04<01:52,  4.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2499/3000 [09:04<01:49,  4.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2503/3000 [09:05<01:17,  6.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▎ | 2505/3000 [09:06<01:45,  4.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▎ | 2506/3000 [09:07<02:42,  3.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▎ | 2508/3000 [09:07<02:08,  3.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▎ | 2512/3000 [09:07<01:16,  6.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2514/3000 [09:08<01:04,  7.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2516/3000 [09:08<01:46,  4.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2518/3000 [09:09<01:38,  4.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2520/3000 [09:09<01:16,  6.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2521/3000 [09:10<02:28,  3.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2522/3000 [09:10<02:29,  3.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2525/3000 [09:11<01:49,  4.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2528/3000 [09:11<01:17,  6.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2530/3000 [09:11<01:12,  6.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2532/3000 [09:12<01:12,  6.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2534/3000 [09:12<01:41,  4.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▍ | 2536/3000 [09:12<01:19,  5.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2538/3000 [09:13<01:06,  7.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2540/3000 [09:13<01:18,  5.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2541/3000 [09:13<01:27,  5.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2542/3000 [09:14<02:12,  3.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2543/3000 [09:14<02:33,  2.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▍ | 2546/3000 [09:15<02:02,  3.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2548/3000 [09:15<01:31,  4.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▌ | 2551/3000 [09:16<01:23,  5.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2554/3000 [09:16<01:07,  6.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2555/3000 [09:17<01:38,  4.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2557/3000 [09:17<01:29,  4.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▌ | 2559/3000 [09:17<01:28,  5.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2560/3000 [09:19<03:02,  2.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▌ | 2562/3000 [09:19<02:30,  2.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2564/3000 [09:19<01:39,  4.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2567/3000 [09:20<01:17,  5.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2569/3000 [09:20<00:58,  7.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2570/3000 [09:20<01:20,  5.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2573/3000 [09:21<01:17,  5.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2575/3000 [09:21<01:28,  4.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2577/3000 [09:22<01:08,  6.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2578/3000 [09:22<01:58,  3.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2579/3000 [09:23<02:55,  2.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2580/3000 [09:23<02:31,  2.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2582/3000 [09:24<02:12,  3.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2584/3000 [09:24<01:36,  4.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2587/3000 [09:24<01:03,  6.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▋ | 2589/3000 [09:25<00:55,  7.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▋ | 2590/3000 [09:25<01:03,  6.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▋ | 2593/3000 [09:25<00:49,  8.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▋ | 2594/3000 [09:25<01:05,  6.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2597/3000 [09:26<00:58,  6.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2598/3000 [09:27<02:30,  2.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2599/3000 [09:28<02:31,  2.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2600/3000 [09:28<02:21,  2.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2601/3000 [09:28<02:23,  2.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2602/3000 [09:28<02:06,  3.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2603/3000 [09:29<02:43,  2.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2606/3000 [09:30<01:59,  3.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2607/3000 [09:30<01:50,  3.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2608/3000 [09:30<01:45,  3.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2610/3000 [09:31<01:24,  4.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2611/3000 [09:31<01:35,  4.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2615/3000 [09:31<00:50,  7.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2617/3000 [09:32<01:03,  6.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2619/3000 [09:32<01:08,  5.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2621/3000 [09:33<01:09,  5.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2622/3000 [09:33<01:45,  3.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2625/3000 [09:34<01:24,  4.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2628/3000 [09:34<01:09,  5.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2630/3000 [09:35<01:13,  5.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2631/3000 [09:35<01:13,  5.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2633/3000 [09:35<01:18,  4.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2635/3000 [09:36<01:35,  3.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2638/3000 [09:37<01:11,  5.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2640/3000 [09:37<01:20,  4.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2641/3000 [09:37<01:14,  4.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2644/3000 [09:38<01:01,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2646/3000 [09:38<01:10,  5.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2647/3000 [09:39<01:22,  4.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2650/3000 [09:39<01:15,  4.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2651/3000 [09:40<01:32,  3.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2652/3000 [09:40<01:42,  3.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2653/3000 [09:41<01:43,  3.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▊ | 2656/3000 [09:41<01:14,  4.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▊ | 2657/3000 [09:41<01:09,  4.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▊ | 2658/3000 [09:42<01:26,  3.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▊ | 2659/3000 [09:42<01:35,  3.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▊ | 2661/3000 [09:43<01:22,  4.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▊ | 2662/3000 [09:43<01:16,  4.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2663/3000 [09:43<01:25,  3.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2664/3000 [09:44<02:11,  2.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2666/3000 [09:44<01:38,  3.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2668/3000 [09:45<01:32,  3.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2670/3000 [09:45<01:21,  4.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2672/3000 [09:45<00:55,  5.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2673/3000 [09:45<00:57,  5.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2675/3000 [09:46<00:53,  6.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2678/3000 [09:46<00:34,  9.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2680/3000 [09:46<00:56,  5.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2681/3000 [09:47<01:12,  4.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2682/3000 [09:47<01:15,  4.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2684/3000 [09:48<01:23,  3.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 2685/3000 [09:48<01:27,  3.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 2686/3000 [09:49<02:04,  2.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|████████▉ | 2688/3000 [09:49<01:26,  3.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|████████▉ | 2691/3000 [09:50<00:59,  5.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 2693/3000 [09:50<00:52,  5.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 2695/3000 [09:50<00:58,  5.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 2696/3000 [09:51<01:07,  4.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|████████▉ | 2699/3000 [09:51<01:07,  4.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2700/3000 [09:52<01:13,  4.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|█████████ | 2703/3000 [09:53<01:17,  3.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2704/3000 [09:53<01:37,  3.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2705/3000 [09:53<01:27,  3.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2706/3000 [09:54<01:29,  3.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|█████████ | 2711/3000 [09:54<00:37,  7.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2713/3000 [09:54<00:40,  7.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2716/3000 [09:55<01:04,  4.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2719/3000 [09:56<00:58,  4.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2720/3000 [09:56<00:53,  5.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2723/3000 [09:57<00:49,  5.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2725/3000 [09:57<00:54,  5.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2726/3000 [09:57<00:50,  5.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2727/3000 [09:58<01:26,  3.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2729/3000 [09:58<01:06,  4.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2730/3000 [09:59<01:04,  4.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2732/3000 [09:59<00:51,  5.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2733/3000 [09:59<00:59,  4.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2736/3000 [10:00<00:44,  5.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████▏| 2738/3000 [10:00<00:44,  5.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████▏| 2742/3000 [10:00<00:38,  6.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████▏| 2744/3000 [10:01<00:59,  4.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2745/3000 [10:01<01:02,  4.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2746/3000 [10:02<01:08,  3.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2749/3000 [10:03<01:01,  4.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2750/3000 [10:03<00:56,  4.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2753/3000 [10:03<00:45,  5.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2757/3000 [10:04<00:31,  7.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2758/3000 [10:04<00:36,  6.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2759/3000 [10:04<00:56,  4.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2761/3000 [10:05<00:50,  4.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2762/3000 [10:05<01:12,  3.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2763/3000 [10:06<01:07,  3.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2767/3000 [10:06<00:45,  5.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2768/3000 [10:06<00:46,  5.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2770/3000 [10:07<00:59,  3.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2772/3000 [10:08<00:53,  4.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2773/3000 [10:08<00:56,  4.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2776/3000 [10:08<00:37,  5.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2780/3000 [10:09<00:28,  7.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2781/3000 [10:09<00:43,  5.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2782/3000 [10:10<00:54,  3.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2784/3000 [10:10<00:48,  4.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2786/3000 [10:11<00:47,  4.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2788/3000 [10:11<00:45,  4.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2792/3000 [10:11<00:34,  6.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2794/3000 [10:12<00:30,  6.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2795/3000 [10:12<00:53,  3.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2797/3000 [10:13<00:57,  3.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2798/3000 [10:13<00:59,  3.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2800/3000 [10:14<00:47,  4.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2802/3000 [10:14<00:38,  5.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2803/3000 [10:14<00:46,  4.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▎| 2805/3000 [10:14<00:37,  5.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▎| 2806/3000 [10:15<00:47,  4.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▎| 2809/3000 [10:15<00:41,  4.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▎| 2810/3000 [10:16<00:38,  4.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▎| 2812/3000 [10:16<00:32,  5.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2813/3000 [10:16<00:35,  5.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2815/3000 [10:17<00:37,  4.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2817/3000 [10:17<00:37,  4.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2818/3000 [10:17<00:43,  4.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2820/3000 [10:18<00:36,  4.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2821/3000 [10:18<00:40,  4.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2823/3000 [10:18<00:31,  5.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2824/3000 [10:18<00:34,  5.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2825/3000 [10:19<01:01,  2.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2827/3000 [10:20<00:47,  3.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2829/3000 [10:20<00:31,  5.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2830/3000 [10:20<00:33,  5.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2831/3000 [10:20<00:33,  5.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2833/3000 [10:20<00:27,  6.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2834/3000 [10:21<00:50,  3.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▍| 2836/3000 [10:22<00:39,  4.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 2837/3000 [10:22<00:58,  2.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 2839/3000 [10:23<00:49,  3.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▍| 2842/3000 [10:23<00:33,  4.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 2844/3000 [10:24<00:32,  4.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▍| 2847/3000 [10:24<00:31,  4.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▍| 2849/3000 [10:25<00:29,  5.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2850/3000 [10:25<00:38,  3.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▌| 2855/3000 [10:25<00:18,  7.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2857/3000 [10:26<00:26,  5.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2858/3000 [10:26<00:33,  4.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2859/3000 [10:27<00:36,  3.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2860/3000 [10:27<00:34,  4.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▌| 2862/3000 [10:27<00:30,  4.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2863/3000 [10:28<00:32,  4.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2866/3000 [10:28<00:28,  4.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2867/3000 [10:29<00:29,  4.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2868/3000 [10:29<00:32,  4.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2872/3000 [10:29<00:17,  7.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2873/3000 [10:30<00:23,  5.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2876/3000 [10:30<00:22,  5.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2878/3000 [10:31<00:24,  4.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2882/3000 [10:31<00:19,  6.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2884/3000 [10:32<00:24,  4.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2885/3000 [10:32<00:24,  4.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▋| 2888/3000 [10:33<00:20,  5.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▋| 2892/3000 [10:33<00:16,  6.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▋| 2894/3000 [10:34<00:19,  5.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▋| 2895/3000 [10:34<00:21,  4.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2898/3000 [10:34<00:18,  5.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2899/3000 [10:35<00:24,  4.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2901/3000 [10:35<00:19,  5.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2902/3000 [10:36<00:23,  4.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2904/3000 [10:36<00:21,  4.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2905/3000 [10:37<00:26,  3.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2907/3000 [10:37<00:23,  3.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2911/3000 [10:37<00:13,  6.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2913/3000 [10:38<00:19,  4.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2914/3000 [10:38<00:19,  4.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2915/3000 [10:39<00:19,  4.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2916/3000 [10:39<00:21,  3.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2918/3000 [10:39<00:18,  4.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2919/3000 [10:40<00:19,  4.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2921/3000 [10:40<00:16,  4.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2923/3000 [10:40<00:13,  5.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2928/3000 [10:41<00:09,  7.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2930/3000 [10:41<00:09,  7.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2931/3000 [10:41<00:14,  4.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2933/3000 [10:42<00:13,  4.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2934/3000 [10:43<00:20,  3.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2936/3000 [10:43<00:15,  4.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2938/3000 [10:43<00:12,  5.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2939/3000 [10:43<00:12,  4.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2940/3000 [10:44<00:12,  4.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2942/3000 [10:44<00:11,  5.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2946/3000 [10:44<00:08,  6.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2949/3000 [10:45<00:05,  8.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2951/3000 [10:45<00:06,  7.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2952/3000 [10:45<00:07,  6.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2953/3000 [10:46<00:09,  5.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2954/3000 [10:46<00:12,  3.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▊| 2956/3000 [10:46<00:10,  4.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▊| 2958/3000 [10:47<00:09,  4.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▊| 2960/3000 [10:47<00:08,  4.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▊| 2962/3000 [10:48<00:08,  4.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2965/3000 [10:48<00:05,  6.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2966/3000 [10:48<00:04,  6.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2969/3000 [10:49<00:04,  6.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2971/3000 [10:50<00:06,  4.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2973/3000 [10:50<00:04,  5.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2974/3000 [10:50<00:06,  3.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2975/3000 [10:50<00:06,  3.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2979/3000 [10:51<00:03,  5.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2981/3000 [10:51<00:03,  4.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2982/3000 [10:52<00:04,  4.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2985/3000 [10:52<00:02,  6.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2986/3000 [10:52<00:02,  5.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2988/3000 [10:53<00:02,  5.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|█████████▉| 2990/3000 [10:53<00:02,  4.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|█████████▉| 2994/3000 [10:54<00:00,  6.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|█████████▉| 2998/3000 [10:54<00:00,  9.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|██████████| 3000/3000 [10:54<00:00,  4.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 08:34:57.469\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m<module>\u001b[0m:\u001b[36m4\u001b[0m - \u001b[1mEvaluation metrics (before optimization): {'f1': 0.0003333333333333333, 'em': 0.0003333333333333333, 'acc': 0.5063333333333333}\u001b[0m\n",
      "\u001b[32m2026-01-13 08:34:57.469\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m<module>\u001b[0m:\u001b[36m6\u001b[0m - \u001b[1mOptimizing workflow...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  0%|          | 0/20 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 08:34:57.473\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2026-01-13 08:35:23.795\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2026-01-13 08:36:59.559\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2026-01-13 08:37:06.538\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2026-01-13 08:37:06.538\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 1 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/150 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   1%|          | 1/150 [00:01<03:49,  1.54s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   1%|▏         | 2/150 [00:02<03:32,  1.44s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   3%|▎         | 5/150 [00:03<01:14,  1.94it/s]\u001b[A\n",
      "Evaluating workflow:   5%|▍         | 7/150 [00:03<00:47,  3.00it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   6%|▌         | 9/150 [00:03<00:37,  3.79it/s]\u001b[A\n",
      "Evaluating workflow:   7%|▋         | 11/150 [00:03<00:27,  5.01it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   9%|▊         | 13/150 [00:04<00:24,  5.50it/s]\u001b[A\n",
      "Evaluating workflow:   9%|▉         | 14/150 [00:04<00:24,  5.67it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  11%|█▏        | 17/150 [00:04<00:20,  6.62it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  13%|█▎        | 19/150 [00:05<00:21,  5.99it/s]\u001b[A\n",
      "Evaluating workflow:  13%|█▎        | 20/150 [00:05<00:21,  6.04it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  15%|█▍        | 22/150 [00:05<00:23,  5.38it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  15%|█▌        | 23/150 [00:06<00:40,  3.11it/s]\u001b[A\n",
      "Evaluating workflow:  16%|█▌        | 24/150 [00:06<00:35,  3.54it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  17%|█▋        | 25/150 [00:06<00:33,  3.79it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  18%|█▊        | 27/150 [00:07<00:32,  3.76it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  19%|█▉        | 29/150 [00:07<00:27,  4.39it/s]\u001b[A\n",
      "Evaluating workflow:  23%|██▎       | 34/150 [00:07<00:13,  8.81it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  24%|██▍       | 36/150 [00:08<00:15,  7.58it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  25%|██▌       | 38/150 [00:08<00:14,  7.95it/s]\u001b[A\n",
      "Evaluating workflow:  27%|██▋       | 40/150 [00:08<00:11,  9.38it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  28%|██▊       | 42/150 [00:09<00:19,  5.56it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  29%|██▉       | 44/150 [00:10<00:29,  3.59it/s]\u001b[A\n",
      "Evaluating workflow:  31%|███       | 46/150 [00:10<00:23,  4.50it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  31%|███▏      | 47/150 [00:11<00:25,  4.06it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  33%|███▎      | 49/150 [00:11<00:22,  4.58it/s]\u001b[A\n",
      "Evaluating workflow:  34%|███▍      | 51/150 [00:11<00:17,  5.82it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  35%|███▍      | 52/150 [00:11<00:18,  5.18it/s]\u001b[A\n",
      "Evaluating workflow:  35%|███▌      | 53/150 [00:11<00:18,  5.36it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  37%|███▋      | 55/150 [00:12<00:14,  6.65it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  37%|███▋      | 56/150 [00:12<00:13,  6.94it/s]\u001b[A\n",
      "Evaluating workflow:  38%|███▊      | 57/150 [00:12<00:14,  6.64it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 59/150 [00:12<00:13,  6.73it/s]\u001b[A\n",
      "Evaluating workflow:  41%|████      | 61/150 [00:12<00:10,  8.50it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  41%|████▏     | 62/150 [00:13<00:12,  7.17it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  43%|████▎     | 64/150 [00:14<00:23,  3.59it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  44%|████▍     | 66/150 [00:14<00:19,  4.35it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  45%|████▍     | 67/150 [00:14<00:24,  3.36it/s]\u001b[A\n",
      "Evaluating workflow:  46%|████▌     | 69/150 [00:15<00:18,  4.46it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  47%|████▋     | 70/150 [00:15<00:18,  4.39it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  48%|████▊     | 72/150 [00:15<00:16,  4.82it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  49%|████▉     | 74/150 [00:16<00:14,  5.20it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  51%|█████     | 76/150 [00:16<00:12,  5.74it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  53%|█████▎    | 79/150 [00:16<00:10,  6.63it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  54%|█████▍    | 81/150 [00:16<00:09,  7.21it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  55%|█████▍    | 82/150 [00:17<00:16,  4.10it/s]\u001b[A\n",
      "Evaluating workflow:  56%|█████▌    | 84/150 [00:17<00:12,  5.47it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  58%|█████▊    | 87/150 [00:18<00:11,  5.54it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  59%|█████▊    | 88/150 [00:18<00:13,  4.66it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  60%|██████    | 90/150 [00:18<00:11,  5.20it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  61%|██████    | 91/150 [00:19<00:16,  3.61it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  61%|██████▏   | 92/150 [00:19<00:15,  3.77it/s]\u001b[A\n",
      "Evaluating workflow:  63%|██████▎   | 95/150 [00:19<00:08,  6.23it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  65%|██████▍   | 97/150 [00:20<00:07,  6.81it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  65%|██████▌   | 98/150 [00:20<00:08,  6.07it/s]\u001b[A\n",
      "Evaluating workflow:  67%|██████▋   | 101/150 [00:20<00:05,  9.22it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  69%|██████▊   | 103/150 [00:21<00:09,  5.11it/s]\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|███████   | 105/150 [00:21<00:07,  5.98it/s]\u001b[A\n",
      "Evaluating workflow:  71%|███████▏  | 107/150 [00:21<00:07,  5.89it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  72%|███████▏  | 108/150 [00:22<00:10,  3.93it/s]\u001b[A\n",
      "Evaluating workflow:  73%|███████▎  | 109/150 [00:22<00:09,  4.21it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  73%|███████▎  | 110/150 [00:22<00:08,  4.80it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  74%|███████▍  | 111/150 [00:23<00:09,  3.98it/s]\u001b[A\n",
      "Evaluating workflow:  75%|███████▍  | 112/150 [00:23<00:08,  4.49it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  77%|███████▋  | 115/150 [00:23<00:04,  7.21it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  77%|███████▋  | 116/150 [00:23<00:04,  7.32it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  79%|███████▊  | 118/150 [00:24<00:04,  6.72it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  79%|███████▉  | 119/150 [00:24<00:05,  5.40it/s]\u001b[A\n",
      "Evaluating workflow:  80%|████████  | 120/150 [00:24<00:05,  5.70it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  81%|████████  | 121/150 [00:24<00:04,  6.13it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  81%|████████▏ | 122/150 [00:24<00:05,  5.07it/s]\u001b[A\n",
      "Evaluating workflow:  83%|████████▎ | 124/150 [00:25<00:03,  7.07it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  83%|████████▎ | 125/150 [00:25<00:03,  7.28it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  84%|████████▍ | 126/150 [00:25<00:06,  3.66it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  85%|████████▍ | 127/150 [00:26<00:06,  3.43it/s]\u001b[A\n",
      "Evaluating workflow:  85%|████████▌ | 128/150 [00:26<00:05,  3.87it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  87%|████████▋ | 130/150 [00:26<00:04,  4.52it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  87%|████████▋ | 131/150 [00:26<00:04,  4.37it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  88%|████████▊ | 132/150 [00:27<00:04,  3.62it/s]\u001b[A\n",
      "Evaluating workflow:  89%|████████▉ | 134/150 [00:27<00:02,  5.48it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  90%|█████████ | 135/150 [00:27<00:03,  4.46it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  91%|█████████ | 136/150 [00:28<00:04,  3.09it/s]\u001b[A\n",
      "Evaluating workflow:  91%|█████████▏| 137/150 [00:28<00:03,  3.74it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  93%|█████████▎| 140/150 [00:28<00:01,  6.90it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  95%|█████████▍| 142/150 [00:29<00:01,  5.06it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  96%|█████████▌| 144/150 [00:29<00:01,  5.49it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  97%|█████████▋| 145/150 [00:29<00:00,  5.10it/s]\u001b[A\n",
      "Evaluating workflow:  98%|█████████▊| 147/150 [00:30<00:00,  6.22it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  99%|█████████▊| 148/150 [00:30<00:00,  5.91it/s]\u001b[A\n",
      "Evaluating workflow:  99%|█████████▉| 149/150 [00:30<00:00,  6.09it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 150/150 [00:30<00:00,  4.86it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 08:37:37.574\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 1 metrics: {'f1': 0.006666666666666667, 'em': 0.006666666666666667, 'acc': 0.49333333333333335}\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      "  5%|▌         | 1/20 [02:40<50:41, 160.10s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 08:37:37.575\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2026-01-13 08:37:58.281\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2026-01-13 08:39:46.702\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2026-01-13 08:39:54.283\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2026-01-13 08:39:54.284\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 2 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/150 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   1%|          | 1/150 [00:01<02:36,  1.05s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   1%|▏         | 2/150 [00:01<02:25,  1.02it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   2%|▏         | 3/150 [00:02<02:22,  1.03it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   3%|▎         | 4/150 [00:03<01:42,  1.43it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   3%|▎         | 5/150 [00:03<01:26,  1.68it/s]\u001b[A\n",
      "Evaluating workflow:   6%|▌         | 9/150 [00:03<00:29,  4.72it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      " {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   7%|▋         | 11/150 [00:03<00:25,  5.48it/s]\u001b[A\n",
      "Evaluating workflow:   9%|▊         | 13/150 [00:04<00:21,  6.48it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  11%|█         | 16/150 [00:04<00:14,  9.43it/s]\u001b[A\n",
      "Evaluating workflow:  13%|█▎        | 19/150 [00:04<00:10, 11.94it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  14%|█▍        | 21/150 [00:05<00:20,  6.16it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  15%|█▌        | 23/150 [00:05<00:27,  4.56it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  17%|█▋        | 25/150 [00:06<00:35,  3.57it/s]\u001b[A\n",
      "Evaluating workflow:  17%|█▋        | 26/150 [00:06<00:32,  3.83it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  18%|█▊        | 27/150 [00:07<00:30,  3.97it/s]\u001b[A\n",
      "Evaluating workflow:  19%|█▊        | 28/150 [00:07<00:27,  4.46it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  20%|██        | 30/150 [00:07<00:21,  5.71it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  21%|██▏       | 32/150 [00:07<00:19,  6.20it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  23%|██▎       | 34/150 [00:08<00:17,  6.62it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  23%|██▎       | 35/150 [00:08<00:18,  6.19it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  25%|██▍       | 37/150 [00:08<00:17,  6.63it/s]\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 40/150 [00:08<00:12,  8.74it/s]\u001b[A\n",
      "Evaluating workflow:  27%|██▋       | 41/150 [00:08<00:15,  7.06it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  28%|██▊       | 42/150 [00:09<00:20,  5.37it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  29%|██▉       | 44/150 [00:09<00:24,  4.34it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  30%|███       | 45/150 [00:10<00:26,  3.91it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  31%|███▏      | 47/150 [00:10<00:23,  4.32it/s]\u001b[A\n",
      "Evaluating workflow:  33%|███▎      | 49/150 [00:10<00:17,  5.63it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  33%|███▎      | 50/150 [00:10<00:17,  5.85it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  35%|███▍      | 52/150 [00:11<00:21,  4.49it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  36%|███▌      | 54/150 [00:11<00:17,  5.48it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  37%|███▋      | 55/150 [00:12<00:22,  4.29it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  37%|███▋      | 56/150 [00:12<00:28,  3.29it/s]\u001b[A\n",
      "Evaluating workflow:  40%|████      | 60/150 [00:13<00:14,  6.23it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  41%|████▏     | 62/150 [00:13<00:12,  7.27it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  43%|████▎     | 64/150 [00:13<00:15,  5.60it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  43%|████▎     | 65/150 [00:13<00:16,  5.28it/s]\u001b[A\n",
      "Evaluating workflow:  44%|████▍     | 66/150 [00:14<00:15,  5.29it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  45%|████▍     | 67/150 [00:14<00:21,  3.84it/s]\u001b[A\n",
      "Evaluating workflow:  46%|████▌     | 69/150 [00:14<00:15,  5.31it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  47%|████▋     | 70/150 [00:15<00:16,  4.97it/s]\u001b[A\n",
      "Evaluating workflow:  49%|████▊     | 73/150 [00:15<00:10,  7.69it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  50%|█████     | 75/150 [00:15<00:08,  8.41it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  51%|█████▏    | 77/150 [00:16<00:15,  4.82it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  52%|█████▏    | 78/150 [00:16<00:16,  4.40it/s]\u001b[A\n",
      "Evaluating workflow:  53%|█████▎    | 79/150 [00:16<00:15,  4.54it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  54%|█████▍    | 81/150 [00:17<00:12,  5.35it/s]\u001b[A\n",
      "Evaluating workflow:  55%|█████▍    | 82/150 [00:17<00:14,  4.78it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  55%|█████▌    | 83/150 [00:17<00:14,  4.73it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  56%|█████▌    | 84/150 [00:17<00:14,  4.44it/s]\u001b[A\n",
      "Evaluating workflow:  57%|█████▋    | 86/150 [00:17<00:10,  5.91it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  58%|█████▊    | 87/150 [00:18<00:10,  6.23it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  59%|█████▊    | 88/150 [00:18<00:11,  5.56it/s]\u001b[A\n",
      "Evaluating workflow:  60%|██████    | 90/150 [00:18<00:07,  7.70it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  61%|██████    | 91/150 [00:18<00:07,  7.64it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  62%|██████▏   | 93/150 [00:18<00:08,  6.99it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  63%|██████▎   | 95/150 [00:19<00:11,  4.86it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  65%|██████▍   | 97/150 [00:19<00:09,  5.68it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  65%|██████▌   | 98/150 [00:20<00:12,  4.13it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  66%|██████▌   | 99/150 [00:20<00:12,  4.10it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  67%|██████▋   | 101/150 [00:21<00:11,  4.12it/s]\u001b[A\n",
      "Evaluating workflow:  69%|██████▊   | 103/150 [00:21<00:08,  5.25it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  70%|███████   | 105/150 [00:21<00:06,  6.67it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  71%|███████   | 106/150 [00:21<00:06,  6.59it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  72%|███████▏  | 108/150 [00:21<00:05,  7.24it/s]\u001b[A\n",
      "Evaluating workflow:  73%|███████▎  | 110/150 [00:21<00:04,  8.93it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  75%|███████▍  | 112/150 [00:22<00:04,  8.73it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  76%|███████▌  | 114/150 [00:22<00:07,  4.96it/s]\u001b[A\n",
      "Evaluating workflow:  77%|███████▋  | 115/150 [00:23<00:06,  5.13it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  77%|███████▋  | 116/150 [00:23<00:08,  3.80it/s]\u001b[A\n",
      "Evaluating workflow:  79%|███████▊  | 118/150 [00:23<00:06,  5.24it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  80%|████████  | 120/150 [00:23<00:04,  6.25it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  81%|████████▏ | 122/150 [00:24<00:04,  6.46it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  82%|████████▏ | 123/150 [00:24<00:06,  3.93it/s]\u001b[A\n",
      "Evaluating workflow:  83%|████████▎ | 124/150 [00:24<00:05,  4.40it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  85%|████████▍ | 127/150 [00:25<00:03,  6.21it/s]\u001b[A\n",
      "Evaluating workflow:  87%|████████▋ | 130/150 [00:25<00:02,  8.88it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  88%|████████▊ | 132/150 [00:25<00:02,  6.08it/s]\u001b[A\n",
      "Evaluating workflow:  89%|████████▉ | 134/150 [00:26<00:02,  7.09it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  91%|█████████ | 136/150 [00:26<00:02,  5.18it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  91%|█████████▏| 137/150 [00:28<00:05,  2.48it/s]\u001b[A\n",
      "Evaluating workflow:  92%|█████████▏| 138/150 [00:28<00:04,  2.83it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  93%|█████████▎| 140/150 [00:28<00:02,  3.63it/s]\u001b[A\n",
      "Evaluating workflow:  94%|█████████▍| 141/150 [00:28<00:02,  4.08it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  95%|█████████▌| 143/150 [00:28<00:01,  5.50it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  98%|█████████▊| 147/150 [00:29<00:00,  5.81it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  99%|█████████▊| 148/150 [00:30<00:00,  4.57it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  99%|█████████▉| 149/150 [00:30<00:00,  4.23it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 150/150 [00:31<00:00,  4.80it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 08:40:25.687\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 2 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.43333333333333335}\u001b[0m\n",
      "\u001b[32m2026-01-13 08:40:25.688\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m291\u001b[0m - \u001b[1mMetrics are worse than the best snapshot which has {'f1': 0.006666666666666667, 'em': 0.006666666666666667, 'acc': 0.49333333333333335}. Rolling back to the best snapshot.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 10%|█         | 2/20 [05:28<49:26, 164.82s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 08:40:25.691\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2026-01-13 08:40:43.707\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2026-01-13 08:42:36.435\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2026-01-13 08:42:45.216\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2026-01-13 08:42:45.217\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 3 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/150 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   1%|          | 1/150 [00:00<02:22,  1.05it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   1%|▏         | 2/150 [00:02<02:32,  1.03s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   2%|▏         | 3/150 [00:03<02:35,  1.06s/it]\u001b[A\n",
      "Evaluating workflow:   3%|▎         | 4/150 [00:03<01:39,  1.47it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   3%|▎         | 5/150 [00:03<01:27,  1.66it/s]\u001b[A\n",
      "Evaluating workflow:   4%|▍         | 6/150 [00:03<01:03,  2.28it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   5%|▍         | 7/150 [00:04<00:53,  2.69it/s]\u001b[A\n",
      "Evaluating workflow:   6%|▌         | 9/150 [00:04<00:33,  4.19it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   8%|▊         | 12/150 [00:04<00:19,  7.05it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   9%|▉         | 14/150 [00:04<00:15,  8.52it/s]\u001b[A\n",
      "Evaluating workflow:  11%|█         | 16/150 [00:04<00:14,  9.57it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  12%|█▏        | 18/150 [00:05<00:16,  7.94it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  13%|█▎        | 20/150 [00:05<00:15,  8.32it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  15%|█▍        | 22/150 [00:05<00:24,  5.23it/s]\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 23/150 [00:06<00:24,  5.18it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  17%|█▋        | 26/150 [00:07<00:34,  3.60it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  18%|█▊        | 27/150 [00:07<00:37,  3.28it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  19%|█▊        | 28/150 [00:07<00:35,  3.47it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  20%|██        | 30/150 [00:08<00:27,  4.37it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  22%|██▏       | 33/150 [00:08<00:20,  5.59it/s]\u001b[A\n",
      "Evaluating workflow:  24%|██▍       | 36/150 [00:08<00:14,  7.70it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  25%|██▌       | 38/150 [00:08<00:12,  9.08it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  27%|██▋       | 40/150 [00:09<00:19,  5.50it/s]\u001b[A\n",
      "Evaluating workflow:  27%|██▋       | 41/150 [00:09<00:19,  5.60it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  29%|██▊       | 43/150 [00:10<00:22,  4.66it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  30%|███       | 45/150 [00:10<00:19,  5.51it/s]\u001b[A\n",
      "Evaluating workflow:  31%|███       | 46/150 [00:10<00:17,  5.80it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  32%|███▏      | 48/150 [00:10<00:15,  6.70it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  33%|███▎      | 49/150 [00:11<00:23,  4.23it/s]\u001b[A\n",
      "Evaluating workflow:  33%|███▎      | 50/150 [00:11<00:21,  4.70it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  34%|███▍      | 51/150 [00:11<00:21,  4.67it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  35%|███▍      | 52/150 [00:12<00:21,  4.63it/s]\u001b[A\n",
      "Evaluating workflow:  35%|███▌      | 53/150 [00:12<00:18,  5.33it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  37%|███▋      | 55/150 [00:12<00:17,  5.51it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  37%|███▋      | 56/150 [00:12<00:18,  5.07it/s]\u001b[A\n",
      "Evaluating workflow:  38%|███▊      | 57/150 [00:12<00:17,  5.23it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  39%|███▊      | 58/150 [00:13<00:18,  4.86it/s]\u001b[A\n",
      "Evaluating workflow:  39%|███▉      | 59/150 [00:13<00:17,  5.23it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  40%|████      | 60/150 [00:13<00:18,  4.87it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  42%|████▏     | 63/150 [00:13<00:12,  6.80it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  43%|████▎     | 64/150 [00:14<00:20,  4.29it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  43%|████▎     | 65/150 [00:14<00:22,  3.70it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  45%|████▌     | 68/150 [00:15<00:16,  5.02it/s]\u001b[A\n",
      "Evaluating workflow:  46%|████▌     | 69/150 [00:15<00:16,  5.04it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  47%|████▋     | 70/150 [00:15<00:17,  4.69it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  48%|████▊     | 72/150 [00:15<00:13,  5.61it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  49%|████▉     | 74/150 [00:16<00:12,  6.30it/s]\u001b[A\n",
      "Evaluating workflow:  50%|█████     | 75/150 [00:16<00:11,  6.39it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  51%|█████     | 76/150 [00:16<00:13,  5.32it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  51%|█████▏    | 77/150 [00:16<00:16,  4.43it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  53%|█████▎    | 79/150 [00:17<00:21,  3.25it/s]\u001b[A\n",
      "Evaluating workflow:  54%|█████▍    | 81/150 [00:17<00:14,  4.61it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  55%|█████▌    | 83/150 [00:18<00:15,  4.21it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  57%|█████▋    | 85/150 [00:18<00:12,  5.06it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  58%|█████▊    | 87/150 [00:19<00:12,  4.97it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  59%|█████▊    | 88/150 [00:19<00:12,  4.82it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  60%|██████    | 90/150 [00:19<00:10,  5.69it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  61%|██████    | 91/150 [00:19<00:10,  5.49it/s]\u001b[A\n",
      "Evaluating workflow:  62%|██████▏   | 93/150 [00:19<00:08,  7.08it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  63%|██████▎   | 94/150 [00:20<00:11,  4.79it/s]\u001b[A\n",
      "Evaluating workflow:  63%|██████▎   | 95/150 [00:20<00:10,  5.09it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  65%|██████▍   | 97/150 [00:20<00:08,  6.01it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  65%|██████▌   | 98/150 [00:20<00:08,  6.04it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  66%|██████▌   | 99/150 [00:21<00:11,  4.40it/s]\u001b[A\n",
      "Evaluating workflow:  67%|██████▋   | 101/150 [00:21<00:08,  5.88it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  69%|██████▊   | 103/150 [00:21<00:08,  5.85it/s]\u001b[A\n",
      "Evaluating workflow:  69%|██████▉   | 104/150 [00:22<00:07,  6.16it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  70%|███████   | 105/150 [00:22<00:10,  4.19it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  71%|███████   | 106/150 [00:22<00:11,  3.86it/s]\u001b[A\n",
      "Evaluating workflow:  71%|███████▏  | 107/150 [00:22<00:10,  4.26it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  73%|███████▎  | 109/150 [00:23<00:06,  6.35it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  74%|███████▍  | 111/150 [00:23<00:09,  4.05it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  75%|███████▌  | 113/150 [00:24<00:07,  4.97it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  77%|███████▋  | 115/150 [00:24<00:06,  5.78it/s]\u001b[A\n",
      "Evaluating workflow:  78%|███████▊  | 117/150 [00:24<00:04,  7.27it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  79%|███████▉  | 119/150 [00:25<00:05,  5.47it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  80%|████████  | 120/150 [00:25<00:05,  5.92it/s]\u001b[A\n",
      "Evaluating workflow:  81%|████████  | 121/150 [00:25<00:04,  6.21it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  81%|████████▏ | 122/150 [00:25<00:06,  4.63it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  82%|████████▏ | 123/150 [00:25<00:06,  4.28it/s]\u001b[A\n",
      "Evaluating workflow:  83%|████████▎ | 125/150 [00:26<00:04,  6.17it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  84%|████████▍ | 126/150 [00:26<00:05,  4.54it/s]\u001b[A\n",
      "Evaluating workflow:  85%|████████▍ | 127/150 [00:26<00:04,  4.75it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  86%|████████▌ | 129/150 [00:27<00:04,  5.24it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  87%|████████▋ | 130/150 [00:27<00:05,  3.94it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  88%|████████▊ | 132/150 [00:27<00:03,  5.06it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  89%|████████▊ | 133/150 [00:28<00:03,  4.42it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  91%|█████████ | 136/150 [00:28<00:02,  6.08it/s]\u001b[A\n",
      "Evaluating workflow:  91%|█████████▏| 137/150 [00:28<00:02,  6.13it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  93%|█████████▎| 139/150 [00:28<00:01,  7.64it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  93%|█████████▎| 140/150 [00:29<00:02,  3.53it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  95%|█████████▍| 142/150 [00:29<00:01,  4.21it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  95%|█████████▌| 143/150 [00:30<00:01,  4.17it/s]\u001b[A\n",
      "Evaluating workflow:  97%|█████████▋| 145/150 [00:30<00:00,  5.62it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  98%|█████████▊| 147/150 [00:30<00:00,  5.13it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  99%|█████████▉| 149/150 [00:31<00:00,  4.21it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 150/150 [00:31<00:00,  4.75it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2026-01-13 08:43:16.903\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 3 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.5533333333333333}\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 15%|█▌        | 3/20 [08:19<47:31, 167.74s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 08:43:16.904\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2026-01-13 08:43:36.771\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2026-01-13 08:45:34.979\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2026-01-13 08:45:46.166\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2026-01-13 08:45:46.167\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 4 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/150 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   1%|          | 1/150 [00:00<02:17,  1.08it/s]\u001b[A\n",
      "Evaluating workflow:   1%|▏         | 2/150 [00:01<01:08,  2.18it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   2%|▏         | 3/150 [00:02<02:08,  1.14it/s]\u001b[A\n",
      "Evaluating workflow:   3%|▎         | 4/150 [00:02<01:24,  1.73it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   4%|▍         | 6/150 [00:03<01:11,  2.00it/s]\u001b[A\n",
      "Evaluating workflow:   5%|▌         | 8/150 [00:03<00:44,  3.17it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   6%|▌         | 9/150 [00:03<00:46,  3.05it/s]\u001b[A\n",
      "Evaluating workflow:   9%|▊         | 13/150 [00:04<00:21,  6.31it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  10%|█         | 15/150 [00:04<00:18,  7.13it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  13%|█▎        | 19/150 [00:04<00:15,  8.61it/s]\u001b[A\n",
      "Evaluating workflow:  15%|█▌        | 23/150 [00:04<00:10, 12.12it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  17%|█▋        | 25/150 [00:05<00:12, 10.33it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  18%|█▊        | 27/150 [00:05<00:12,  9.56it/s]\u001b[A\n",
      "Evaluating workflow:  19%|█▉        | 29/150 [00:05<00:12,  9.96it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  21%|██        | 31/150 [00:06<00:23,  5.16it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  21%|██▏       | 32/150 [00:06<00:24,  4.85it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  22%|██▏       | 33/150 [00:07<00:34,  3.42it/s]\u001b[A\n",
      "Evaluating workflow:  23%|██▎       | 34/150 [00:07<00:29,  3.89it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  23%|██▎       | 35/150 [00:07<00:27,  4.14it/s]\u001b[A\n",
      "Evaluating workflow:  24%|██▍       | 36/150 [00:07<00:25,  4.40it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  25%|██▌       | 38/150 [00:08<00:20,  5.56it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  26%|██▌       | 39/150 [00:08<00:21,  5.08it/s]\u001b[A\n",
      "Evaluating workflow:  27%|██▋       | 41/150 [00:08<00:16,  6.49it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  28%|██▊       | 42/150 [00:08<00:17,  6.16it/s]\u001b[A\n",
      "Evaluating workflow:  29%|██▉       | 44/150 [00:08<00:13,  8.15it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  32%|███▏      | 48/150 [00:09<00:10,  9.67it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  33%|███▎      | 50/150 [00:09<00:11,  8.58it/s]\u001b[A\n",
      "Evaluating workflow:  35%|███▌      | 53/150 [00:09<00:08, 11.43it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  37%|███▋      | 55/150 [00:09<00:09, 10.33it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  38%|███▊      | 57/150 [00:09<00:08, 11.21it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  39%|███▉      | 59/150 [00:11<00:23,  3.94it/s]\u001b[A\n",
      "Evaluating workflow:  40%|████      | 60/150 [00:11<00:21,  4.14it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  41%|████      | 61/150 [00:11<00:19,  4.55it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  42%|████▏     | 63/150 [00:11<00:16,  5.29it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  43%|████▎     | 64/150 [00:12<00:18,  4.56it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  45%|████▍     | 67/150 [00:12<00:16,  4.93it/s]\u001b[A\n",
      "Evaluating workflow:  46%|████▌     | 69/150 [00:12<00:13,  6.15it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  47%|████▋     | 71/150 [00:12<00:10,  7.79it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  49%|████▊     | 73/150 [00:13<00:11,  6.76it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  50%|█████     | 75/150 [00:13<00:08,  8.42it/s]\u001b[A\n",
      "Evaluating workflow:  52%|█████▏    | 78/150 [00:13<00:06, 11.14it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  53%|█████▎    | 80/150 [00:14<00:12,  5.48it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  55%|█████▍    | 82/150 [00:14<00:12,  5.25it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  55%|█████▌    | 83/150 [00:15<00:14,  4.48it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  56%|█████▌    | 84/150 [00:15<00:15,  4.34it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  57%|█████▋    | 86/150 [00:15<00:13,  4.76it/s]\u001b[A\n",
      "Evaluating workflow:  58%|█████▊    | 87/150 [00:16<00:13,  4.82it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  59%|█████▊    | 88/150 [00:16<00:13,  4.54it/s]\u001b[A\n",
      "Evaluating workflow:  62%|██████▏   | 93/150 [00:16<00:06,  9.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[A\n",
      "Evaluating workflow:  63%|██████▎   | 95/150 [00:16<00:05,  9.46it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  65%|██████▍   | 97/150 [00:17<00:08,  6.61it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  67%|██████▋   | 100/150 [00:17<00:06,  8.31it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  68%|██████▊   | 102/150 [00:17<00:05,  8.16it/s]\u001b[A\n",
      "Evaluating workflow:  69%|██████▊   | 103/150 [00:17<00:05,  7.88it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  69%|██████▉   | 104/150 [00:18<00:07,  5.83it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  70%|███████   | 105/150 [00:18<00:08,  5.25it/s]\u001b[A\n",
      "Evaluating workflow:  71%|███████   | 106/150 [00:18<00:08,  5.32it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  72%|███████▏  | 108/150 [00:19<00:07,  5.63it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  73%|███████▎  | 109/150 [00:19<00:11,  3.61it/s]\u001b[A\n",
      "Evaluating workflow:  73%|███████▎  | 110/150 [00:19<00:09,  4.18it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  74%|███████▍  | 111/150 [00:20<00:09,  3.95it/s]\u001b[A\n",
      "Evaluating workflow:  75%|███████▍  | 112/150 [00:20<00:08,  4.66it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  77%|███████▋  | 115/150 [00:20<00:04,  8.18it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  78%|███████▊  | 117/150 [00:20<00:04,  7.12it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  79%|███████▊  | 118/150 [00:20<00:05,  6.33it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  79%|███████▉  | 119/150 [00:21<00:06,  4.88it/s]\u001b[A\n",
      "Evaluating workflow:  80%|████████  | 120/150 [00:21<00:05,  5.39it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  81%|████████▏ | 122/150 [00:21<00:05,  5.41it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  83%|████████▎ | 124/150 [00:22<00:04,  6.03it/s]\u001b[A\n",
      "Evaluating workflow:  84%|████████▍ | 126/150 [00:22<00:03,  7.56it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  85%|████████▌ | 128/150 [00:22<00:03,  6.84it/s]\u001b[A\n",
      "Evaluating workflow:  86%|████████▌ | 129/150 [00:22<00:02,  7.23it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  87%|████████▋ | 131/150 [00:22<00:02,  7.95it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  88%|████████▊ | 132/150 [00:23<00:03,  4.89it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  89%|████████▊ | 133/150 [00:23<00:04,  4.20it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  89%|████████▉ | 134/150 [00:23<00:03,  4.30it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  91%|█████████▏| 137/150 [00:24<00:02,  6.38it/s]\u001b[A\n",
      "Evaluating workflow:  92%|█████████▏| 138/150 [00:24<00:01,  6.29it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  93%|█████████▎| 139/150 [00:25<00:03,  3.30it/s]\u001b[A\n",
      "Evaluating workflow:  94%|█████████▍| 141/150 [00:25<00:01,  4.55it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  95%|█████████▌| 143/150 [00:25<00:01,  6.17it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  97%|█████████▋| 145/150 [00:25<00:00,  6.20it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  99%|█████████▉| 149/150 [00:26<00:00,  7.63it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 150/150 [00:26<00:00,  5.69it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 08:46:12.640\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 4 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.4666666666666667}\u001b[0m\n",
      "\u001b[32m2026-01-13 08:46:12.640\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m291\u001b[0m - \u001b[1mMetrics are worse than the best snapshot which has {'f1': 0.0, 'em': 0.0, 'acc': 0.5533333333333333}. Rolling back to the best snapshot.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 20%|██        | 4/20 [11:15<45:34, 170.90s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 08:46:12.644\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2026-01-13 08:46:33.859\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2026-01-13 08:48:20.046\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2026-01-13 08:48:30.850\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2026-01-13 08:48:30.850\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 5 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/150 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   1%|          | 1/150 [00:01<04:24,  1.78s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   1%|▏         | 2/150 [00:02<03:31,  1.43s/it]\u001b[A\n",
      "Evaluating workflow:   2%|▏         | 3/150 [00:03<02:00,  1.22it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   3%|▎         | 4/150 [00:03<01:46,  1.37it/s]\u001b[A\n",
      "Evaluating workflow:   4%|▍         | 6/150 [00:03<00:54,  2.65it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   7%|▋         | 10/150 [00:03<00:23,  6.03it/s]\u001b[A\n",
      "Evaluating workflow:   9%|▊         | 13/150 [00:04<00:15,  8.58it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  11%|█         | 16/150 [00:04<00:17,  7.75it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  12%|█▏        | 18/150 [00:04<00:20,  6.50it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  13%|█▎        | 20/150 [00:05<00:23,  5.44it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  14%|█▍        | 21/150 [00:06<00:48,  2.68it/s]\u001b[A\n",
      "Evaluating workflow:  15%|█▍        | 22/150 [00:06<00:42,  3.04it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  15%|█▌        | 23/150 [00:07<00:44,  2.84it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  17%|█▋        | 25/150 [00:07<00:36,  3.41it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  18%|█▊        | 27/150 [00:08<00:29,  4.20it/s]\u001b[A\n",
      "Evaluating workflow:  19%|█▊        | 28/150 [00:08<00:25,  4.72it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  19%|█▉        | 29/150 [00:08<00:24,  4.98it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  21%|██        | 31/150 [00:08<00:24,  4.94it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  23%|██▎       | 34/150 [00:09<00:19,  5.80it/s]\u001b[A\n",
      "Evaluating workflow:  23%|██▎       | 35/150 [00:09<00:19,  5.76it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  25%|██▌       | 38/150 [00:09<00:14,  7.72it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  26%|██▌       | 39/150 [00:09<00:14,  7.91it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  27%|██▋       | 41/150 [00:10<00:30,  3.54it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  28%|██▊       | 42/150 [00:11<00:30,  3.56it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  29%|██▊       | 43/150 [00:11<00:30,  3.51it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  30%|███       | 45/150 [00:11<00:23,  4.51it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  31%|███       | 46/150 [00:11<00:24,  4.25it/s]\u001b[A\n",
      "Evaluating workflow:  32%|███▏      | 48/150 [00:12<00:18,  5.49it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  34%|███▍      | 51/150 [00:12<00:11,  8.26it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  35%|███▌      | 53/150 [00:13<00:21,  4.55it/s]\u001b[A\n",
      "Evaluating workflow:  37%|███▋      | 55/150 [00:13<00:16,  5.92it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  38%|███▊      | 57/150 [00:13<00:18,  5.15it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  39%|███▉      | 59/150 [00:13<00:13,  6.54it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  41%|████      | 61/150 [00:14<00:15,  5.72it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  42%|████▏     | 63/150 [00:14<00:13,  6.67it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  43%|████▎     | 65/150 [00:15<00:15,  5.43it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  44%|████▍     | 66/150 [00:15<00:15,  5.31it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  45%|████▍     | 67/150 [00:15<00:17,  4.78it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  45%|████▌     | 68/150 [00:15<00:20,  4.03it/s]\u001b[A\n",
      "Evaluating workflow:  47%|████▋     | 70/150 [00:16<00:14,  5.36it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  47%|████▋     | 71/150 [00:16<00:14,  5.60it/s]\u001b[A\n",
      "Evaluating workflow:  48%|████▊     | 72/150 [00:16<00:13,  5.78it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  49%|████▉     | 74/150 [00:16<00:16,  4.62it/s]\u001b[A\n",
      "Evaluating workflow:  50%|█████     | 75/150 [00:17<00:14,  5.25it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  51%|█████     | 76/150 [00:17<00:13,  5.41it/s]\u001b[A\n",
      "Evaluating workflow:  51%|█████▏    | 77/150 [00:17<00:13,  5.38it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  52%|█████▏    | 78/150 [00:17<00:12,  5.68it/s]\u001b[A\n",
      "Evaluating workflow:  53%|█████▎    | 79/150 [00:17<00:11,  6.39it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  53%|█████▎    | 80/150 [00:17<00:11,  6.15it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  55%|█████▌    | 83/150 [00:18<00:12,  5.28it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  57%|█████▋    | 86/150 [00:18<00:09,  7.00it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  58%|█████▊    | 87/150 [00:19<00:13,  4.71it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  59%|█████▊    | 88/150 [00:19<00:14,  4.35it/s]\u001b[A\n",
      "Evaluating workflow:  59%|█████▉    | 89/150 [00:19<00:12,  4.88it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  60%|██████    | 90/150 [00:19<00:10,  5.56it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  61%|██████    | 91/150 [00:20<00:11,  5.23it/s]\u001b[A\n",
      "Evaluating workflow:  61%|██████▏   | 92/150 [00:20<00:10,  5.76it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  62%|██████▏   | 93/150 [00:20<00:12,  4.55it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  63%|██████▎   | 94/150 [00:20<00:13,  4.03it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  65%|██████▍   | 97/150 [00:21<00:12,  4.30it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  65%|██████▌   | 98/150 [00:21<00:13,  3.91it/s]\u001b[A\n",
      "Evaluating workflow:  66%|██████▌   | 99/150 [00:21<00:11,  4.38it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  68%|██████▊   | 102/150 [00:22<00:06,  7.25it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  69%|██████▉   | 104/150 [00:22<00:06,  7.28it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  71%|███████   | 106/150 [00:22<00:05,  7.46it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  71%|███████▏  | 107/150 [00:22<00:07,  5.94it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  72%|███████▏  | 108/150 [00:23<00:09,  4.47it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  73%|███████▎  | 109/150 [00:23<00:08,  4.56it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  73%|███████▎  | 110/150 [00:23<00:09,  4.33it/s]\u001b[A\n",
      "Evaluating workflow:  74%|███████▍  | 111/150 [00:23<00:07,  4.92it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  76%|███████▌  | 114/150 [00:24<00:08,  4.29it/s]\u001b[A\n",
      "Evaluating workflow:  77%|███████▋  | 115/150 [00:24<00:07,  4.52it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  78%|███████▊  | 117/150 [00:25<00:05,  6.10it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  79%|███████▉  | 119/150 [00:25<00:04,  7.03it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  81%|████████  | 121/150 [00:25<00:04,  6.40it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  81%|████████▏ | 122/150 [00:26<00:06,  4.18it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  83%|████████▎ | 125/150 [00:26<00:04,  5.47it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  85%|████████▍ | 127/150 [00:26<00:03,  5.90it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  85%|████████▌ | 128/150 [00:27<00:04,  5.34it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  87%|████████▋ | 130/150 [00:27<00:05,  3.87it/s]\u001b[A\n",
      "Evaluating workflow:  87%|████████▋ | 131/150 [00:28<00:04,  4.36it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  88%|████████▊ | 132/150 [00:28<00:05,  3.06it/s]\u001b[A\n",
      "Evaluating workflow:  90%|█████████ | 135/150 [00:28<00:02,  5.21it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  91%|█████████▏| 137/150 [00:29<00:02,  6.33it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  93%|█████████▎| 139/150 [00:29<00:01,  6.91it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  94%|█████████▍| 141/150 [00:29<00:01,  7.19it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  95%|█████████▌| 143/150 [00:29<00:00,  7.24it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  97%|█████████▋| 145/150 [00:30<00:00,  6.84it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  97%|█████████▋| 146/150 [00:30<00:00,  5.65it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  98%|█████████▊| 147/150 [00:30<00:00,  4.36it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  99%|█████████▊| 148/150 [00:31<00:00,  3.71it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 150/150 [00:31<00:00,  4.73it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 08:49:02.688\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 5 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.43333333333333335}\u001b[0m\n",
      "\u001b[32m2026-01-13 08:49:02.688\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m291\u001b[0m - \u001b[1mMetrics are worse than the best snapshot which has {'f1': 0.0, 'em': 0.0, 'acc': 0.5533333333333333}. Rolling back to the best snapshot.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 25%|██▌       | 5/20 [14:05<42:38, 170.59s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 08:49:02.692\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2026-01-13 08:49:24.110\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2026-01-13 08:51:18.148\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2026-01-13 08:51:26.701\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2026-01-13 08:51:26.701\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 6 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/150 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   1%|          | 1/150 [00:01<04:31,  1.82s/it]\u001b[A\n",
      "Evaluating workflow:   1%|▏         | 2/150 [00:01<02:02,  1.21it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   2%|▏         | 3/150 [00:03<02:33,  1.04s/it]\u001b[A\n",
      "Evaluating workflow:   3%|▎         | 4/150 [00:03<01:40,  1.45it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   4%|▍         | 6/150 [00:03<00:52,  2.77it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   7%|▋         | 10/150 [00:03<00:24,  5.73it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   8%|▊         | 12/150 [00:04<00:23,  5.80it/s]\u001b[A\n",
      "Evaluating workflow:   9%|▉         | 14/150 [00:04<00:19,  7.03it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  12%|█▏        | 18/150 [00:04<00:13,  9.97it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  13%|█▎        | 20/150 [00:04<00:12, 10.52it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  15%|█▍        | 22/150 [00:04<00:13,  9.83it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 24/150 [00:06<00:31,  4.03it/s]\u001b[A\n",
      "Evaluating workflow:  17%|█▋        | 25/150 [00:06<00:42,  2.97it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  18%|█▊        | 27/150 [00:07<00:36,  3.41it/s]\u001b[A\n",
      "Evaluating workflow:  19%|█▊        | 28/150 [00:07<00:33,  3.67it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  21%|██        | 31/150 [00:07<00:20,  5.85it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  22%|██▏       | 33/150 [00:07<00:20,  5.83it/s]\u001b[A\n",
      "Evaluating workflow:  23%|██▎       | 34/150 [00:08<00:18,  6.18it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  23%|██▎       | 35/150 [00:08<00:17,  6.39it/s]\u001b[A\n",
      "Evaluating workflow:  25%|██▍       | 37/150 [00:08<00:14,  7.69it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  27%|██▋       | 40/150 [00:08<00:10, 10.13it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  29%|██▊       | 43/150 [00:08<00:08, 13.34it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  31%|███       | 46/150 [00:10<00:25,  4.13it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  32%|███▏      | 48/150 [00:11<00:31,  3.26it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  33%|███▎      | 49/150 [00:11<00:29,  3.42it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  34%|███▍      | 51/150 [00:12<00:28,  3.52it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  36%|███▌      | 54/150 [00:12<00:19,  4.92it/s]\u001b[A\n",
      "Evaluating workflow:  39%|███▊      | 58/150 [00:12<00:12,  7.37it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  40%|████      | 60/150 [00:12<00:11,  7.90it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  41%|████▏     | 62/150 [00:13<00:16,  5.39it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  43%|████▎     | 64/150 [00:13<00:15,  5.65it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  44%|████▍     | 66/150 [00:14<00:17,  4.77it/s]\u001b[A\n",
      "Evaluating workflow:  45%|████▍     | 67/150 [00:14<00:16,  4.99it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  45%|████▌     | 68/150 [00:14<00:20,  4.02it/s]\u001b[A\n",
      "Evaluating workflow:  47%|████▋     | 70/150 [00:15<00:14,  5.40it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  47%|████▋     | 71/150 [00:15<00:14,  5.44it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  48%|████▊     | 72/150 [00:15<00:24,  3.20it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  49%|████▉     | 74/150 [00:16<00:23,  3.24it/s]\u001b[A\n",
      "Evaluating workflow:  50%|█████     | 75/150 [00:16<00:19,  3.76it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  52%|█████▏    | 78/150 [00:17<00:17,  4.22it/s]\u001b[A\n",
      "Evaluating workflow:  54%|█████▍    | 81/150 [00:17<00:11,  6.25it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  56%|█████▌    | 84/150 [00:17<00:08,  7.82it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  57%|█████▋    | 86/150 [00:18<00:14,  4.33it/s]\u001b[A\n",
      "Evaluating workflow:  58%|█████▊    | 87/150 [00:18<00:14,  4.50it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  59%|█████▊    | 88/150 [00:19<00:12,  4.84it/s]\u001b[A\n",
      "Evaluating workflow:  60%|██████    | 90/150 [00:19<00:09,  6.44it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  61%|██████▏   | 92/150 [00:19<00:13,  4.38it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  62%|██████▏   | 93/150 [00:20<00:14,  3.82it/s]\u001b[A\n",
      "Evaluating workflow:  63%|██████▎   | 95/150 [00:20<00:10,  5.01it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  65%|██████▍   | 97/150 [00:20<00:09,  5.80it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  65%|██████▌   | 98/150 [00:21<00:10,  4.80it/s]\u001b[A\n",
      "Evaluating workflow:  66%|██████▌   | 99/150 [00:21<00:10,  5.08it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  68%|██████▊   | 102/150 [00:21<00:06,  6.90it/s]\u001b[A\n",
      "Evaluating workflow:  69%|██████▊   | 103/150 [00:21<00:07,  6.61it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  70%|███████   | 105/150 [00:21<00:05,  7.60it/s]\u001b[A\n",
      "Evaluating workflow:  71%|███████   | 106/150 [00:21<00:05,  7.63it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  72%|███████▏  | 108/150 [00:22<00:10,  4.18it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  73%|███████▎  | 109/150 [00:23<00:12,  3.35it/s]\u001b[A\n",
      "Evaluating workflow:  73%|███████▎  | 110/150 [00:23<00:10,  3.72it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  74%|███████▍  | 111/150 [00:23<00:09,  4.11it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  76%|███████▌  | 114/150 [00:24<00:06,  5.59it/s]\u001b[A\n",
      "Evaluating workflow:  77%|███████▋  | 115/150 [00:24<00:06,  5.61it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  77%|███████▋  | 116/150 [00:24<00:05,  5.69it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  79%|███████▊  | 118/150 [00:24<00:05,  5.68it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  80%|████████  | 120/150 [00:25<00:04,  6.22it/s]\u001b[A\n",
      "Evaluating workflow:  81%|████████  | 121/150 [00:25<00:04,  6.01it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  81%|████████▏ | 122/150 [00:25<00:04,  6.22it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  82%|████████▏ | 123/150 [00:25<00:06,  4.15it/s]\u001b[A\n",
      "Evaluating workflow:  83%|████████▎ | 124/150 [00:26<00:05,  4.36it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  83%|████████▎ | 125/150 [00:26<00:09,  2.74it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  85%|████████▍ | 127/150 [00:27<00:06,  3.60it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  85%|████████▌ | 128/150 [00:27<00:06,  3.16it/s]\u001b[A\n",
      "Evaluating workflow:  86%|████████▌ | 129/150 [00:27<00:05,  3.77it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  87%|████████▋ | 130/150 [00:27<00:05,  3.93it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  87%|████████▋ | 131/150 [00:28<00:06,  2.89it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  88%|████████▊ | 132/150 [00:28<00:06,  2.74it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  89%|████████▉ | 134/150 [00:29<00:04,  3.81it/s]\u001b[A\n",
      "Evaluating workflow:  90%|█████████ | 135/150 [00:29<00:03,  4.31it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  92%|█████████▏| 138/150 [00:29<00:01,  6.46it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  93%|█████████▎| 140/150 [00:29<00:01,  7.09it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  95%|█████████▍| 142/150 [00:30<00:01,  7.06it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  95%|█████████▌| 143/150 [00:30<00:01,  6.26it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  96%|█████████▌| 144/150 [00:30<00:01,  5.31it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  98%|█████████▊| 147/150 [00:30<00:00,  6.28it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  99%|█████████▊| 148/150 [00:31<00:00,  3.35it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  99%|█████████▉| 149/150 [00:32<00:00,  2.56it/s]\u001b[A\n",
      "Evaluating workflow: 100%|██████████| 150/150 [00:32<00:00,  4.58it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 08:51:59.579\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 6 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.4866666666666667}\u001b[0m\n",
      "\u001b[32m2026-01-13 08:51:59.580\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m291\u001b[0m - \u001b[1mMetrics are worse than the best snapshot which has {'f1': 0.0, 'em': 0.0, 'acc': 0.5533333333333333}. Rolling back to the best snapshot.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 30%|███       | 6/20 [17:02<40:18, 172.73s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 08:51:59.583\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2026-01-13 08:52:22.894\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2026-01-13 08:54:27.488\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2026-01-13 08:54:38.520\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2026-01-13 08:54:38.520\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 7 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/150 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   1%|          | 1/150 [00:01<04:03,  1.64s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   1%|▏         | 2/150 [00:02<02:55,  1.19s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   3%|▎         | 4/150 [00:02<01:21,  1.78it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   3%|▎         | 5/150 [00:03<01:10,  2.06it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   5%|▍         | 7/150 [00:03<00:51,  2.75it/s]\u001b[A\n",
      "Evaluating workflow:   6%|▌         | 9/150 [00:03<00:35,  3.92it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   7%|▋         | 11/150 [00:03<00:25,  5.45it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   9%|▉         | 14/150 [00:04<00:17,  7.84it/s]\u001b[A\n",
      "Evaluating workflow:  11%|█         | 16/150 [00:04<00:15,  8.82it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  12%|█▏        | 18/150 [00:04<00:18,  7.09it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  15%|█▍        | 22/150 [00:05<00:19,  6.49it/s]\u001b[A\n",
      "Evaluating workflow:  15%|█▌        | 23/150 [00:05<00:19,  6.45it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  17%|█▋        | 25/150 [00:05<00:19,  6.56it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  17%|█▋        | 26/150 [00:06<00:21,  5.64it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  18%|█▊        | 27/150 [00:06<00:30,  4.01it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  19%|█▊        | 28/150 [00:07<00:35,  3.41it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  19%|█▉        | 29/150 [00:07<00:33,  3.60it/s]\u001b[A\n",
      "Evaluating workflow:  21%|██▏       | 32/150 [00:07<00:19,  6.09it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  23%|██▎       | 35/150 [00:07<00:13,  8.48it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  25%|██▍       | 37/150 [00:08<00:15,  7.49it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  26%|██▌       | 39/150 [00:08<00:15,  7.26it/s]\u001b[A\n",
      "Evaluating workflow:  27%|██▋       | 41/150 [00:08<00:12,  8.80it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  29%|██▉       | 44/150 [00:09<00:19,  5.32it/s]\u001b[A\n",
      "Evaluating workflow:  31%|███       | 46/150 [00:09<00:15,  6.60it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  32%|███▏      | 48/150 [00:10<00:19,  5.20it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  33%|███▎      | 50/150 [00:10<00:16,  6.22it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  35%|███▍      | 52/150 [00:10<00:17,  5.65it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  35%|███▌      | 53/150 [00:11<00:21,  4.50it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  37%|███▋      | 56/150 [00:11<00:14,  6.28it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  39%|███▊      | 58/150 [00:11<00:16,  5.65it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  41%|████▏     | 62/150 [00:12<00:11,  7.70it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  42%|████▏     | 63/150 [00:12<00:13,  6.35it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  44%|████▍     | 66/150 [00:12<00:12,  6.74it/s]\u001b[A\n",
      "Evaluating workflow:  45%|████▍     | 67/150 [00:12<00:12,  6.64it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  45%|████▌     | 68/150 [00:13<00:16,  4.98it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  47%|████▋     | 70/150 [00:13<00:13,  5.75it/s]\u001b[A\n",
      "Evaluating workflow:  48%|████▊     | 72/150 [00:13<00:10,  7.11it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  49%|████▊     | 73/150 [00:13<00:10,  7.09it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  50%|█████     | 75/150 [00:14<00:18,  4.15it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  51%|█████     | 76/150 [00:15<00:18,  3.93it/s]\u001b[A\n",
      "Evaluating workflow:  51%|█████▏    | 77/150 [00:15<00:17,  4.24it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  53%|█████▎    | 79/150 [00:15<00:12,  5.79it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  54%|█████▍    | 81/150 [00:15<00:11,  6.12it/s]\u001b[A\n",
      "Evaluating workflow:  55%|█████▌    | 83/150 [00:15<00:08,  7.82it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 85/150 [00:16<00:11,  5.71it/s]\u001b[A\n",
      "Evaluating workflow:  58%|█████▊    | 87/150 [00:16<00:08,  7.22it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  59%|█████▉    | 89/150 [00:16<00:09,  6.48it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  60%|██████    | 90/150 [00:17<00:10,  5.80it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  61%|██████    | 91/150 [00:17<00:10,  5.60it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  61%|██████▏   | 92/150 [00:17<00:12,  4.59it/s]\u001b[A\n",
      "Evaluating workflow:  62%|██████▏   | 93/150 [00:17<00:11,  5.10it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  63%|██████▎   | 95/150 [00:17<00:08,  6.73it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  65%|██████▍   | 97/150 [00:18<00:09,  5.50it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  66%|██████▌   | 99/150 [00:18<00:08,  5.74it/s]\u001b[A\n",
      "Evaluating workflow:  67%|██████▋   | 100/150 [00:18<00:08,  6.24it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  67%|██████▋   | 101/150 [00:19<00:10,  4.67it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  68%|██████▊   | 102/150 [00:19<00:10,  4.40it/s]\u001b[A\n",
      "Evaluating workflow:  69%|██████▉   | 104/150 [00:19<00:07,  6.39it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  70%|███████   | 105/150 [00:19<00:06,  6.51it/s]\u001b[A\n",
      "Evaluating workflow:  71%|███████   | 106/150 [00:19<00:07,  6.21it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  72%|███████▏  | 108/150 [00:20<00:05,  7.63it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  73%|███████▎  | 109/150 [00:20<00:06,  6.55it/s]\u001b[A\n",
      "Evaluating workflow:  73%|███████▎  | 110/150 [00:20<00:05,  7.02it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  74%|███████▍  | 111/150 [00:20<00:08,  4.44it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  76%|███████▌  | 114/150 [00:21<00:06,  5.95it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  77%|███████▋  | 115/150 [00:21<00:06,  5.29it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  77%|███████▋  | 116/150 [00:22<00:09,  3.74it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  78%|███████▊  | 117/150 [00:22<00:09,  3.58it/s]\u001b[A\n",
      "Evaluating workflow:  79%|███████▊  | 118/150 [00:22<00:07,  4.25it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  79%|███████▉  | 119/150 [00:22<00:07,  4.29it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  80%|████████  | 120/150 [00:23<00:09,  3.06it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  82%|████████▏ | 123/150 [00:23<00:05,  4.76it/s]\u001b[A\n",
      "Evaluating workflow:  85%|████████▍ | 127/150 [00:23<00:02,  8.32it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  86%|████████▌ | 129/150 [00:24<00:02,  7.40it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  88%|████████▊ | 132/150 [00:24<00:02,  6.97it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  89%|████████▊ | 133/150 [00:24<00:03,  5.53it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  89%|████████▉ | 134/150 [00:25<00:03,  4.86it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  91%|█████████ | 136/150 [00:25<00:02,  4.91it/s]\u001b[A\n",
      "Evaluating workflow:  91%|█████████▏| 137/150 [00:25<00:02,  5.18it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  92%|█████████▏| 138/150 [00:26<00:02,  4.73it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  93%|█████████▎| 140/150 [00:27<00:03,  3.11it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  94%|█████████▍| 141/150 [00:27<00:02,  3.36it/s]\u001b[A\n",
      "Evaluating workflow:  95%|█████████▍| 142/150 [00:27<00:02,  3.84it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  96%|█████████▌| 144/150 [00:27<00:01,  4.00it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  97%|█████████▋| 146/150 [00:28<00:00,  4.94it/s]\u001b[A\n",
      "Evaluating workflow:  98%|█████████▊| 147/150 [00:28<00:00,  5.39it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  99%|█████████▊| 148/150 [00:28<00:00,  5.48it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  99%|█████████▉| 149/150 [00:29<00:00,  2.66it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 150/150 [00:29<00:00,  5.04it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 08:55:08.397\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 7 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.5133333333333333}\u001b[0m\n",
      "\u001b[32m2026-01-13 08:55:08.398\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m291\u001b[0m - \u001b[1mMetrics are worse than the best snapshot which has {'f1': 0.0, 'em': 0.0, 'acc': 0.5533333333333333}. Rolling back to the best snapshot.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 35%|███▌      | 7/20 [20:10<38:33, 177.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 08:55:08.401\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2026-01-13 08:55:27.245\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2026-01-13 08:57:16.317\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2026-01-13 08:57:29.023\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2026-01-13 08:57:29.024\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 8 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/150 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   1%|          | 1/150 [00:01<03:54,  1.57s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   1%|▏         | 2/150 [00:01<01:53,  1.30it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   3%|▎         | 4/150 [00:02<01:17,  1.87it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   3%|▎         | 5/150 [00:03<01:30,  1.60it/s]\u001b[A\n",
      "Evaluating workflow:   5%|▍         | 7/150 [00:03<00:50,  2.81it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   6%|▌         | 9/150 [00:03<00:35,  4.00it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   7%|▋         | 11/150 [00:03<00:29,  4.67it/s]\u001b[A\n",
      "Evaluating workflow:   9%|▉         | 14/150 [00:04<00:19,  6.86it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  12%|█▏        | 18/150 [00:04<00:14,  9.28it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  13%|█▎        | 20/150 [00:04<00:13,  9.33it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  15%|█▍        | 22/150 [00:04<00:13,  9.84it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  16%|█▌        | 24/150 [00:05<00:20,  6.26it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  18%|█▊        | 27/150 [00:06<00:23,  5.15it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  19%|█▊        | 28/150 [00:06<00:33,  3.60it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  19%|█▉        | 29/150 [00:07<00:33,  3.63it/s]\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██▏       | 32/150 [00:07<00:21,  5.38it/s]\u001b[A\n",
      "Evaluating workflow:  22%|██▏       | 33/150 [00:07<00:25,  4.50it/s]\u001b[A\n",
      "Evaluating workflow:  23%|██▎       | 35/150 [00:07<00:19,  5.95it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  26%|██▌       | 39/150 [00:08<00:12,  8.91it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  27%|██▋       | 41/150 [00:08<00:14,  7.47it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  29%|██▊       | 43/150 [00:08<00:12,  8.87it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  30%|███       | 45/150 [00:09<00:15,  6.76it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  31%|███▏      | 47/150 [00:09<00:18,  5.63it/s]\u001b[A\n",
      "Evaluating workflow:  33%|███▎      | 49/150 [00:09<00:14,  7.09it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  34%|███▍      | 51/150 [00:09<00:13,  7.11it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  35%|███▌      | 53/150 [00:10<00:19,  4.95it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  36%|███▌      | 54/150 [00:11<00:24,  3.95it/s]\u001b[A\n",
      "Evaluating workflow:  37%|███▋      | 56/150 [00:11<00:18,  5.18it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  38%|███▊      | 57/150 [00:11<00:20,  4.55it/s]\u001b[A\n",
      "Evaluating workflow:  39%|███▉      | 59/150 [00:11<00:14,  6.09it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  41%|████      | 61/150 [00:12<00:12,  6.90it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  43%|████▎     | 64/150 [00:12<00:09,  9.13it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  45%|████▌     | 68/150 [00:12<00:08, 10.20it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  47%|████▋     | 70/150 [00:13<00:11,  6.86it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  47%|████▋     | 71/150 [00:13<00:11,  7.17it/s]\u001b[A\n",
      "Evaluating workflow:  48%|████▊     | 72/150 [00:13<00:10,  7.14it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  49%|████▊     | 73/150 [00:13<00:11,  6.87it/s]\u001b[A\n",
      "Evaluating workflow:  49%|████▉     | 74/150 [00:13<00:11,  6.76it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  51%|█████     | 76/150 [00:13<00:09,  7.70it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  52%|█████▏    | 78/150 [00:14<00:13,  5.54it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  53%|█████▎    | 79/150 [00:14<00:13,  5.28it/s]\u001b[A\n",
      "Evaluating workflow:  53%|█████▎    | 80/150 [00:14<00:13,  5.33it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  54%|█████▍    | 81/150 [00:14<00:12,  5.67it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  55%|█████▍    | 82/150 [00:15<00:13,  4.89it/s]\u001b[A\n",
      "Evaluating workflow:  56%|█████▌    | 84/150 [00:15<00:09,  7.16it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  57%|█████▋    | 86/150 [00:15<00:12,  5.08it/s]\u001b[A\n",
      "Evaluating workflow:  59%|█████▊    | 88/150 [00:16<00:09,  6.32it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  59%|█████▉    | 89/150 [00:16<00:09,  6.33it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  60%|██████    | 90/150 [00:16<00:12,  4.67it/s]\u001b[A\n",
      "Evaluating workflow:  62%|██████▏   | 93/150 [00:16<00:07,  7.86it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  64%|██████▍   | 96/150 [00:17<00:05,  9.72it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  65%|██████▌   | 98/150 [00:17<00:06,  8.31it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  67%|██████▋   | 100/150 [00:17<00:05,  8.41it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  68%|██████▊   | 102/150 [00:18<00:09,  5.26it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  69%|██████▊   | 103/150 [00:18<00:09,  5.01it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  69%|██████▉   | 104/150 [00:18<00:10,  4.27it/s]\u001b[A\n",
      "Evaluating workflow:  71%|███████   | 106/150 [00:19<00:07,  5.51it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  71%|███████▏  | 107/150 [00:19<00:08,  5.31it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  73%|███████▎  | 109/150 [00:19<00:08,  5.12it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  73%|███████▎  | 110/150 [00:20<00:10,  3.98it/s]\u001b[A\n",
      "Evaluating workflow:  75%|███████▍  | 112/150 [00:20<00:07,  5.37it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  76%|███████▌  | 114/150 [00:20<00:05,  7.16it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  77%|███████▋  | 116/150 [00:20<00:04,  7.80it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  79%|███████▊  | 118/150 [00:20<00:04,  7.93it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  79%|███████▉  | 119/150 [00:21<00:04,  6.42it/s]\u001b[A\n",
      "Evaluating workflow:  80%|████████  | 120/150 [00:21<00:04,  6.21it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  81%|████████▏ | 122/150 [00:21<00:03,  7.07it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  82%|████████▏ | 123/150 [00:21<00:03,  7.14it/s]\u001b[A\n",
      "Evaluating workflow:  83%|████████▎ | 124/150 [00:21<00:03,  6.88it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  84%|████████▍ | 126/150 [00:21<00:02,  9.04it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  85%|████████▌ | 128/150 [00:22<00:04,  4.94it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  86%|████████▌ | 129/150 [00:23<00:04,  4.42it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  87%|████████▋ | 130/150 [00:23<00:05,  3.96it/s]\u001b[A\n",
      "Evaluating workflow:  87%|████████▋ | 131/150 [00:23<00:04,  4.26it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  89%|████████▊ | 133/150 [00:23<00:03,  4.58it/s]\u001b[A\n",
      "Evaluating workflow:  89%|████████▉ | 134/150 [00:24<00:03,  5.14it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  91%|█████████ | 136/150 [00:24<00:02,  6.67it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  91%|█████████▏| 137/150 [00:24<00:02,  4.76it/s]\u001b[A\n",
      "Evaluating workflow:  93%|█████████▎| 139/150 [00:24<00:01,  6.09it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  93%|█████████▎| 140/150 [00:24<00:01,  6.42it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  95%|█████████▌| 143/150 [00:25<00:00,  7.39it/s]\u001b[A\n",
      "Evaluating workflow:  96%|█████████▌| 144/150 [00:25<00:00,  7.44it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  97%|█████████▋| 145/150 [00:25<00:00,  6.99it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  97%|█████████▋| 146/150 [00:25<00:00,  6.06it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  98%|█████████▊| 147/150 [00:26<00:00,  3.99it/s]\u001b[A\n",
      "Evaluating workflow:  99%|█████████▊| 148/150 [00:26<00:00,  4.64it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  99%|█████████▉| 149/150 [00:26<00:00,  3.60it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 150/150 [00:27<00:00,  5.45it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 08:57:56.635\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 8 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.5666666666666667}\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 40%|████      | 8/20 [22:59<34:58, 174.89s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 08:57:56.636\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2026-01-13 08:58:14.554\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2026-01-13 09:00:05.102\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2026-01-13 09:00:17.953\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2026-01-13 09:00:17.953\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 9 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/150 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   1%|          | 1/150 [00:01<03:42,  1.49s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   1%|▏         | 2/150 [00:01<01:59,  1.24it/s]\u001b[A\n",
      "Evaluating workflow:   3%|▎         | 4/150 [00:01<00:50,  2.88it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   3%|▎         | 5/150 [00:02<00:46,  3.09it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   4%|▍         | 6/150 [00:03<01:20,  1.79it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   6%|▌         | 9/150 [00:03<00:41,  3.42it/s]\u001b[A\n",
      "Evaluating workflow:   8%|▊         | 12/150 [00:03<00:25,  5.35it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   9%|▉         | 14/150 [00:04<00:23,  5.85it/s]\u001b[A\n",
      "Evaluating workflow:  11%|█         | 16/150 [00:04<00:18,  7.40it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  12%|█▏        | 18/150 [00:04<00:17,  7.54it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  13%|█▎        | 20/150 [00:04<00:16,  8.10it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  15%|█▍        | 22/150 [00:04<00:17,  7.30it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  16%|█▌        | 24/150 [00:05<00:16,  7.86it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  17%|█▋        | 25/150 [00:05<00:18,  6.91it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  17%|█▋        | 26/150 [00:06<00:33,  3.69it/s]\u001b[A\n",
      "Evaluating workflow:  18%|█▊        | 27/150 [00:06<00:29,  4.12it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  19%|█▊        | 28/150 [00:06<00:30,  4.05it/s]\u001b[A\n",
      "Evaluating workflow:  19%|█▉        | 29/150 [00:06<00:27,  4.41it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  20%|██        | 30/150 [00:06<00:23,  5.17it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  21%|██        | 31/150 [00:07<00:28,  4.15it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  22%|██▏       | 33/150 [00:07<00:26,  4.46it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  23%|██▎       | 35/150 [00:08<00:32,  3.51it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  25%|██▍       | 37/150 [00:08<00:25,  4.47it/s]\u001b[A\n",
      "Evaluating workflow:  25%|██▌       | 38/150 [00:08<00:22,  4.91it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  26%|██▌       | 39/150 [00:08<00:20,  5.29it/s]\u001b[A\n",
      "Evaluating workflow:  27%|██▋       | 40/150 [00:08<00:19,  5.67it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  27%|██▋       | 41/150 [00:09<00:17,  6.29it/s]\u001b[A\n",
      "Evaluating workflow:  29%|██▊       | 43/150 [00:09<00:12,  8.78it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  30%|███       | 45/150 [00:09<00:10, 10.33it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      " {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  31%|███▏      | 47/150 [00:09<00:18,  5.58it/s]\u001b[A\n",
      "Evaluating workflow:  32%|███▏      | 48/150 [00:10<00:17,  5.71it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  33%|███▎      | 49/150 [00:10<00:25,  3.88it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  33%|███▎      | 50/150 [00:11<00:29,  3.34it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  34%|███▍      | 51/150 [00:11<00:32,  3.07it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  35%|███▍      | 52/150 [00:11<00:29,  3.33it/s]\u001b[A\n",
      "Evaluating workflow:  37%|███▋      | 55/150 [00:11<00:15,  6.02it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  37%|███▋      | 56/150 [00:12<00:20,  4.55it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  39%|███▊      | 58/150 [00:12<00:20,  4.44it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  40%|████      | 60/150 [00:13<00:16,  5.46it/s]\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 61/150 [00:13<00:16,  5.35it/s]\u001b[A\n",
      "Evaluating workflow:  41%|████▏     | 62/150 [00:13<00:15,  5.54it/s]\u001b[A\n",
      "Evaluating workflow:  43%|████▎     | 64/150 [00:13<00:11,  7.63it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  44%|████▍     | 66/150 [00:13<00:09,  8.83it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  45%|████▌     | 68/150 [00:14<00:11,  7.04it/s]\u001b[A\n",
      "Evaluating workflow:  47%|████▋     | 70/150 [00:14<00:09,  8.03it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  49%|████▊     | 73/150 [00:14<00:13,  5.63it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  50%|█████     | 75/150 [00:15<00:14,  5.28it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  51%|█████     | 76/150 [00:15<00:14,  5.11it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  52%|█████▏    | 78/150 [00:15<00:13,  5.53it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  53%|█████▎    | 79/150 [00:16<00:14,  5.03it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  53%|█████▎    | 80/150 [00:16<00:14,  4.86it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  55%|█████▍    | 82/150 [00:16<00:14,  4.64it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  56%|█████▌    | 84/150 [00:17<00:11,  5.69it/s]\u001b[A\n",
      "Evaluating workflow:  57%|█████▋    | 85/150 [00:17<00:10,  6.15it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  57%|█████▋    | 86/150 [00:17<00:18,  3.47it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  59%|█████▊    | 88/150 [00:18<00:14,  4.14it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  59%|█████▉    | 89/150 [00:18<00:15,  3.83it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  60%|██████    | 90/150 [00:18<00:15,  3.84it/s]\u001b[A\n",
      "Evaluating workflow:  62%|██████▏   | 93/150 [00:18<00:08,  6.75it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  63%|██████▎   | 95/150 [00:19<00:07,  7.36it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  65%|██████▍   | 97/150 [00:19<00:08,  6.23it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  66%|██████▌   | 99/150 [00:19<00:07,  7.01it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  67%|██████▋   | 100/150 [00:20<00:11,  4.18it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  67%|██████▋   | 101/150 [00:20<00:11,  4.24it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  68%|██████▊   | 102/150 [00:20<00:11,  4.32it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  69%|██████▊   | 103/150 [00:21<00:12,  3.68it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  69%|██████▉   | 104/150 [00:21<00:11,  3.89it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  71%|███████▏  | 107/150 [00:21<00:07,  6.08it/s]\u001b[A\n",
      "Evaluating workflow:  72%|███████▏  | 108/150 [00:21<00:07,  5.99it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  73%|███████▎  | 109/150 [00:22<00:09,  4.37it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  73%|███████▎  | 110/150 [00:22<00:09,  4.41it/s]\u001b[A\n",
      "Evaluating workflow:  75%|███████▍  | 112/150 [00:22<00:06,  6.21it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  75%|███████▌  | 113/150 [00:23<00:08,  4.12it/s]\u001b[A\n",
      "Evaluating workflow:  77%|███████▋  | 115/150 [00:23<00:05,  5.96it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  78%|███████▊  | 117/150 [00:23<00:04,  7.03it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  79%|███████▉  | 119/150 [00:24<00:05,  5.61it/s]\u001b[A\n",
      "Evaluating workflow:  80%|████████  | 120/150 [00:24<00:05,  5.89it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  81%|████████  | 121/150 [00:24<00:06,  4.42it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  81%|████████▏ | 122/150 [00:24<00:06,  4.49it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  84%|████████▍ | 126/150 [00:25<00:03,  6.04it/s]\u001b[A\n",
      "Evaluating workflow:  85%|████████▍ | 127/150 [00:25<00:03,  5.93it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  85%|████████▌ | 128/150 [00:25<00:03,  5.58it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  86%|████████▌ | 129/150 [00:26<00:04,  5.23it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  87%|████████▋ | 130/150 [00:26<00:04,  4.70it/s]\u001b[A\n",
      "Evaluating workflow:  88%|████████▊ | 132/150 [00:26<00:02,  6.38it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  89%|████████▊ | 133/150 [00:27<00:04,  3.85it/s]\u001b[A\n",
      "Evaluating workflow:  89%|████████▉ | 134/150 [00:27<00:03,  4.41it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  91%|█████████ | 136/150 [00:27<00:02,  5.95it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  91%|█████████▏| 137/150 [00:27<00:02,  4.57it/s]\u001b[A\n",
      "Evaluating workflow:  92%|█████████▏| 138/150 [00:27<00:02,  4.69it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  94%|█████████▍| 141/150 [00:28<00:01,  5.87it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  95%|█████████▌| 143/150 [00:28<00:01,  6.09it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  96%|█████████▌| 144/150 [00:28<00:01,  5.12it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  97%|█████████▋| 146/150 [00:29<00:01,  3.73it/s]\u001b[A\n",
      "Evaluating workflow:  98%|█████████▊| 147/150 [00:29<00:00,  4.05it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  99%|█████████▊| 148/150 [00:30<00:00,  4.01it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  99%|█████████▉| 149/150 [00:30<00:00,  3.72it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 150/150 [00:32<00:00,  4.55it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 09:00:51.018\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 9 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.5266666666666666}\u001b[0m\n",
      "\u001b[32m2026-01-13 09:00:51.020\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m291\u001b[0m - \u001b[1mMetrics are worse than the best snapshot which has {'f1': 0.0, 'em': 0.0, 'acc': 0.5666666666666667}. Rolling back to the best snapshot.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 45%|████▌     | 9/20 [25:53<32:02, 174.73s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 09:00:51.022\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2026-01-13 09:01:16.020\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2026-01-13 09:03:13.373\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2026-01-13 09:03:25.606\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2026-01-13 09:03:25.606\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 10 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/150 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   1%|          | 1/150 [00:01<04:33,  1.83s/it]\u001b[A\n",
      "Evaluating workflow:   1%|▏         | 2/150 [00:01<02:05,  1.18it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   2%|▏         | 3/150 [00:02<01:17,  1.89it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   3%|▎         | 5/150 [00:03<01:23,  1.73it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   4%|▍         | 6/150 [00:03<01:12,  1.99it/s]\u001b[A\n",
      "Evaluating workflow:   5%|▍         | 7/150 [00:03<00:58,  2.44it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   5%|▌         | 8/150 [00:04<00:49,  2.85it/s]\u001b[A\n",
      "Evaluating workflow:   6%|▌         | 9/150 [00:04<00:41,  3.43it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   7%|▋         | 11/150 [00:04<00:27,  5.14it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   9%|▊         | 13/150 [00:04<00:31,  4.34it/s]\u001b[A\n",
      "Evaluating workflow:   9%|▉         | 14/150 [00:05<00:28,  4.82it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  11%|█         | 16/150 [00:05<00:21,  6.30it/s]\u001b[A\n",
      "Evaluating workflow:  11%|█▏        | 17/150 [00:05<00:21,  6.29it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  13%|█▎        | 19/150 [00:05<00:15,  8.51it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  15%|█▍        | 22/150 [00:05<00:17,  7.42it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  18%|█▊        | 27/150 [00:06<00:13,  9.32it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  19%|█▉        | 29/150 [00:07<00:22,  5.33it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  20%|██        | 30/150 [00:07<00:26,  4.54it/s]\u001b[A\n",
      "Evaluating workflow:  21%|██        | 31/150 [00:07<00:23,  4.98it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  21%|██▏       | 32/150 [00:07<00:22,  5.20it/s]\u001b[A\n",
      "Evaluating workflow:  22%|██▏       | 33/150 [00:08<00:20,  5.75it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  24%|██▍       | 36/150 [00:08<00:14,  8.04it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  25%|██▌       | 38/150 [00:08<00:15,  7.38it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  26%|██▌       | 39/150 [00:08<00:18,  5.98it/s]\u001b[A\n",
      "Evaluating workflow:  28%|██▊       | 42/150 [00:09<00:12,  8.47it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  29%|██▉       | 44/150 [00:09<00:11,  9.20it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  31%|███       | 46/150 [00:09<00:17,  6.03it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  31%|███▏      | 47/150 [00:10<00:18,  5.54it/s]\u001b[A\n",
      "Evaluating workflow:  32%|███▏      | 48/150 [00:10<00:18,  5.53it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  33%|███▎      | 50/150 [00:10<00:16,  6.14it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  34%|███▍      | 51/150 [00:10<00:15,  6.41it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  35%|███▌      | 53/150 [00:11<00:18,  5.31it/s]\u001b[A\n",
      "Evaluating workflow:  36%|███▌      | 54/150 [00:11<00:17,  5.64it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  37%|███▋      | 55/150 [00:11<00:17,  5.35it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  38%|███▊      | 57/150 [00:11<00:15,  5.93it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  39%|███▉      | 59/150 [00:12<00:14,  6.15it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  41%|████      | 61/150 [00:12<00:13,  6.41it/s]\u001b[A\n",
      "Evaluating workflow:  42%|████▏     | 63/150 [00:12<00:11,  7.35it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  43%|████▎     | 64/150 [00:12<00:12,  6.71it/s]\u001b[A\n",
      "Evaluating workflow:  43%|████▎     | 65/150 [00:12<00:13,  6.49it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  44%|████▍     | 66/150 [00:13<00:13,  6.31it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  45%|████▍     | 67/150 [00:13<00:16,  4.93it/s]\u001b[A\n",
      "Evaluating workflow:  46%|████▌     | 69/150 [00:13<00:11,  7.02it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  47%|████▋     | 70/150 [00:13<00:11,  7.20it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  49%|████▊     | 73/150 [00:14<00:11,  6.96it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  49%|████▉     | 74/150 [00:14<00:16,  4.60it/s]\u001b[A\n",
      "Evaluating workflow:  51%|█████     | 76/150 [00:14<00:11,  6.25it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  52%|█████▏    | 78/150 [00:15<00:10,  6.65it/s]\u001b[A\n",
      "Evaluating workflow:  53%|█████▎    | 79/150 [00:15<00:10,  6.79it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  54%|█████▍    | 81/150 [00:15<00:09,  7.55it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  55%|█████▍    | 82/150 [00:15<00:10,  6.21it/s]\u001b[A\n",
      "Evaluating workflow:  57%|█████▋    | 85/150 [00:15<00:06,  9.79it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  58%|█████▊    | 87/150 [00:16<00:10,  5.96it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  59%|█████▉    | 89/150 [00:16<00:09,  6.30it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  61%|██████    | 91/150 [00:17<00:12,  4.90it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  63%|██████▎   | 94/150 [00:17<00:08,  6.65it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  63%|██████▎   | 95/150 [00:17<00:09,  5.78it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  64%|██████▍   | 96/150 [00:18<00:10,  5.33it/s]\u001b[A\n",
      "Evaluating workflow:  65%|██████▍   | 97/150 [00:18<00:09,  5.78it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  65%|██████▌   | 98/150 [00:18<00:08,  5.78it/s]\u001b[A\n",
      "Evaluating workflow:  66%|██████▌   | 99/150 [00:18<00:07,  6.44it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  67%|██████▋   | 100/150 [00:18<00:08,  6.24it/s]\u001b[A\n",
      "Evaluating workflow:  69%|██████▊   | 103/150 [00:18<00:04, 10.45it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|███████   | 105/150 [00:19<00:06,  7.27it/s]\u001b[A\n",
      "Evaluating workflow:  71%|███████▏  | 107/150 [00:19<00:06,  6.88it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  72%|███████▏  | 108/150 [00:19<00:05,  7.26it/s]\u001b[A\n",
      "Evaluating workflow:  73%|███████▎  | 109/150 [00:19<00:05,  6.88it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  74%|███████▍  | 111/150 [00:20<00:06,  6.03it/s]\u001b[A\n",
      "Evaluating workflow:  75%|███████▍  | 112/150 [00:20<00:05,  6.37it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  75%|███████▌  | 113/150 [00:20<00:05,  6.65it/s]\u001b[A\n",
      "Evaluating workflow:  77%|███████▋  | 115/150 [00:20<00:03,  9.06it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  78%|███████▊  | 117/150 [00:21<00:05,  5.97it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  79%|███████▉  | 119/150 [00:21<00:07,  4.40it/s]\u001b[A\n",
      "Evaluating workflow:  80%|████████  | 120/150 [00:21<00:06,  4.74it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  81%|████████  | 121/150 [00:22<00:05,  5.10it/s]\u001b[A\n",
      "Evaluating workflow:  81%|████████▏ | 122/150 [00:22<00:04,  5.69it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  82%|████████▏ | 123/150 [00:22<00:04,  6.13it/s]\u001b[A\n",
      "Evaluating workflow:  83%|████████▎ | 124/150 [00:22<00:04,  6.40it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  84%|████████▍ | 126/150 [00:22<00:04,  5.18it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  87%|████████▋ | 131/150 [00:23<00:01,  9.53it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  89%|████████▊ | 133/150 [00:23<00:01,  9.13it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  90%|█████████ | 135/150 [00:23<00:01,  7.58it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  91%|█████████ | 136/150 [00:23<00:01,  7.55it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  91%|█████████▏| 137/150 [00:24<00:02,  6.42it/s]\u001b[A\n",
      "Evaluating workflow:  94%|█████████▍| 141/150 [00:24<00:00, 10.63it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  95%|█████████▌| 143/150 [00:24<00:00,  7.54it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  97%|█████████▋| 145/150 [00:25<00:01,  3.86it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  97%|█████████▋| 146/150 [00:26<00:01,  3.17it/s]\u001b[A\n",
      "Evaluating workflow:  98%|█████████▊| 147/150 [00:26<00:00,  3.50it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  99%|█████████▊| 148/150 [00:26<00:00,  3.91it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 150/150 [00:27<00:00,  5.46it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 09:03:53.184\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 10 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.5333333333333333}\u001b[0m\n",
      "\u001b[32m2026-01-13 09:03:53.185\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m291\u001b[0m - \u001b[1mMetrics are worse than the best snapshot which has {'f1': 0.0, 'em': 0.0, 'acc': 0.5666666666666667}. Rolling back to the best snapshot.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 50%|█████     | 10/20 [28:55<29:30, 177.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 09:03:53.188\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2026-01-13 09:04:12.867\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2026-01-13 09:06:14.518\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2026-01-13 09:06:26.768\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2026-01-13 09:06:26.768\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 11 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/150 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   1%|          | 1/150 [00:01<04:56,  1.99s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   2%|▏         | 3/150 [00:02<01:26,  1.69it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   3%|▎         | 4/150 [00:02<01:22,  1.77it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   5%|▍         | 7/150 [00:03<00:44,  3.21it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   7%|▋         | 10/150 [00:03<00:28,  4.84it/s]\u001b[A\n",
      "Evaluating workflow:   8%|▊         | 12/150 [00:03<00:22,  6.19it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 14/150 [00:03<00:21,  6.26it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  11%|█         | 16/150 [00:04<00:21,  6.34it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  13%|█▎        | 19/150 [00:04<00:16,  7.87it/s]\u001b[A\n",
      "Evaluating workflow:  15%|█▍        | 22/150 [00:04<00:12, 10.51it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  16%|█▌        | 24/150 [00:05<00:26,  4.83it/s]\u001b[A\n",
      "Evaluating workflow:  17%|█▋        | 26/150 [00:05<00:21,  5.77it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  19%|█▊        | 28/150 [00:06<00:21,  5.65it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  19%|█▉        | 29/150 [00:06<00:20,  5.89it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  20%|██        | 30/150 [00:06<00:25,  4.68it/s]\u001b[A\n",
      "Evaluating workflow:  21%|██        | 31/150 [00:06<00:25,  4.76it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  21%|██▏       | 32/150 [00:07<00:28,  4.17it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  25%|██▍       | 37/150 [00:07<00:17,  6.45it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  25%|██▌       | 38/150 [00:07<00:19,  5.74it/s]\u001b[A\n",
      "Evaluating workflow:  26%|██▌       | 39/150 [00:08<00:18,  6.15it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  27%|██▋       | 40/150 [00:08<00:17,  6.23it/s]\u001b[A\n",
      "Evaluating workflow:  29%|██▊       | 43/150 [00:08<00:10,  9.82it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  30%|███       | 45/150 [00:08<00:12,  8.28it/s]\u001b[A\n",
      "Evaluating workflow:  31%|███▏      | 47/150 [00:08<00:10,  9.39it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  33%|███▎      | 49/150 [00:10<00:32,  3.10it/s]\u001b[A\n",
      "Evaluating workflow:  33%|███▎      | 50/150 [00:10<00:28,  3.51it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  35%|███▍      | 52/150 [00:10<00:20,  4.81it/s]\u001b[A\n",
      "Evaluating workflow:  37%|███▋      | 55/150 [00:10<00:13,  7.21it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  38%|███▊      | 57/150 [00:11<00:14,  6.57it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  39%|███▉      | 59/150 [00:11<00:12,  7.00it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  41%|████      | 61/150 [00:11<00:12,  7.30it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  42%|████▏     | 63/150 [00:11<00:11,  7.60it/s]\u001b[A\n",
      "Evaluating workflow:  43%|████▎     | 64/150 [00:12<00:12,  7.09it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  43%|████▎     | 65/150 [00:12<00:12,  7.01it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  44%|████▍     | 66/150 [00:12<00:17,  4.84it/s]\u001b[A\n",
      "Evaluating workflow:  45%|████▍     | 67/150 [00:12<00:16,  5.18it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  46%|████▌     | 69/150 [00:13<00:15,  5.39it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  47%|████▋     | 70/150 [00:13<00:27,  2.96it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  47%|████▋     | 71/150 [00:14<00:24,  3.28it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  48%|████▊     | 72/150 [00:14<00:27,  2.88it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  49%|████▉     | 74/150 [00:14<00:19,  3.98it/s]\u001b[A\n",
      "Evaluating workflow:  51%|█████▏    | 77/150 [00:15<00:10,  6.71it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  53%|█████▎    | 79/150 [00:15<00:09,  7.75it/s]\u001b[A\n",
      "Evaluating workflow:  54%|█████▍    | 81/150 [00:15<00:07,  9.47it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  55%|█████▌    | 83/150 [00:15<00:06,  9.70it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 85/150 [00:16<00:10,  6.41it/s]\u001b[A\n",
      "Evaluating workflow:  58%|█████▊    | 87/150 [00:16<00:08,  7.13it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  59%|█████▊    | 88/150 [00:16<00:11,  5.43it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  59%|█████▉    | 89/150 [00:17<00:14,  4.28it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  60%|██████    | 90/150 [00:17<00:18,  3.22it/s]\u001b[A\n",
      "Evaluating workflow:  61%|██████▏   | 92/150 [00:17<00:12,  4.64it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  62%|██████▏   | 93/150 [00:18<00:13,  4.22it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  63%|██████▎   | 94/150 [00:18<00:16,  3.46it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  64%|██████▍   | 96/150 [00:18<00:11,  4.55it/s]\u001b[A\n",
      "Evaluating workflow:  65%|██████▍   | 97/150 [00:18<00:11,  4.66it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  65%|██████▌   | 98/150 [00:19<00:10,  5.06it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  66%|██████▌   | 99/150 [00:19<00:11,  4.58it/s]\u001b[A\n",
      "Evaluating workflow:  67%|██████▋   | 100/150 [00:19<00:09,  5.12it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  69%|██████▉   | 104/150 [00:19<00:04, 10.44it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  71%|███████▏  | 107/150 [00:19<00:03, 11.79it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  73%|███████▎  | 109/150 [00:20<00:05,  7.53it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  74%|███████▍  | 111/150 [00:20<00:05,  7.47it/s]\u001b[A\n",
      "Evaluating workflow:  75%|███████▍  | 112/150 [00:20<00:05,  7.43it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  75%|███████▌  | 113/150 [00:21<00:05,  6.45it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  77%|███████▋  | 116/150 [00:21<00:05,  5.89it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  78%|███████▊  | 117/150 [00:22<00:07,  4.25it/s]\u001b[A\n",
      "Evaluating workflow:  79%|███████▊  | 118/150 [00:22<00:06,  4.69it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  79%|███████▉  | 119/150 [00:22<00:09,  3.13it/s]\u001b[A\n",
      "Evaluating workflow:  81%|████████  | 121/150 [00:23<00:06,  4.45it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  81%|████████▏ | 122/150 [00:23<00:06,  4.01it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  82%|████████▏ | 123/150 [00:23<00:07,  3.68it/s]\u001b[A\n",
      "Evaluating workflow:  83%|████████▎ | 124/150 [00:23<00:06,  4.13it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  83%|████████▎ | 125/150 [00:24<00:05,  4.53it/s]\u001b[A\n",
      "Evaluating workflow:  87%|████████▋ | 130/150 [00:24<00:01, 10.95it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  89%|████████▊ | 133/150 [00:24<00:01, 10.20it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  91%|█████████ | 136/150 [00:25<00:02,  6.64it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  92%|█████████▏| 138/150 [00:25<00:01,  6.77it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  93%|█████████▎| 140/150 [00:26<00:02,  3.57it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  95%|█████████▍| 142/150 [00:27<00:02,  3.78it/s]\u001b[A\n",
      "Evaluating workflow:  95%|█████████▌| 143/150 [00:27<00:01,  4.03it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  96%|█████████▌| 144/150 [00:27<00:01,  3.76it/s]\u001b[A\n",
      "Evaluating workflow:  97%|█████████▋| 145/150 [00:27<00:01,  4.31it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  97%|█████████▋| 146/150 [00:28<00:00,  4.83it/s]\u001b[A\n",
      "Evaluating workflow:  98%|█████████▊| 147/150 [00:28<00:00,  5.39it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  99%|█████████▉| 149/150 [00:29<00:00,  3.36it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 150/150 [00:29<00:00,  5.03it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2026-01-13 09:06:56.703\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 11 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.5266666666666666}\u001b[0m\n",
      "\u001b[32m2026-01-13 09:06:56.703\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m291\u001b[0m - \u001b[1mMetrics are worse than the best snapshot which has {'f1': 0.0, 'em': 0.0, 'acc': 0.5666666666666667}. Rolling back to the best snapshot.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 55%|█████▌    | 11/20 [31:59<26:51, 179.01s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 09:06:56.706\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2026-01-13 09:07:18.630\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2026-01-13 09:09:15.841\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2026-01-13 09:09:29.115\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2026-01-13 09:09:29.115\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 12 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/150 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   1%|          | 1/150 [00:01<03:22,  1.36s/it]\u001b[A\n",
      "Evaluating workflow:   1%|▏         | 2/150 [00:01<01:37,  1.52it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   3%|▎         | 4/150 [00:01<00:41,  3.53it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   4%|▍         | 6/150 [00:02<00:40,  3.58it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   5%|▍         | 7/150 [00:02<00:41,  3.45it/s]\u001b[A\n",
      "Evaluating workflow:   5%|▌         | 8/150 [00:02<00:35,  4.03it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   6%|▌         | 9/150 [00:02<00:32,  4.32it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   7%|▋         | 10/150 [00:03<00:44,  3.13it/s]\u001b[A\n",
      "Evaluating workflow:   9%|▊         | 13/150 [00:03<00:24,  5.56it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   9%|▉         | 14/150 [00:03<00:23,  5.82it/s]\u001b[A\n",
      "Evaluating workflow:  12%|█▏        | 18/150 [00:03<00:12, 10.69it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  13%|█▎        | 20/150 [00:03<00:10, 11.86it/s]\u001b[A\n",
      "Evaluating workflow:  15%|█▌        | 23/150 [00:04<00:09, 14.09it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  17%|█▋        | 25/150 [00:04<00:10, 11.94it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  18%|█▊        | 27/150 [00:04<00:16,  7.57it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  19%|█▉        | 29/150 [00:05<00:21,  5.74it/s]\u001b[A\n",
      "Evaluating workflow:  21%|██        | 31/150 [00:05<00:17,  6.81it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  22%|██▏       | 33/150 [00:05<00:15,  7.33it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  23%|██▎       | 35/150 [00:06<00:26,  4.33it/s]\u001b[A\n",
      "Evaluating workflow:  25%|██▍       | 37/150 [00:06<00:21,  5.37it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  25%|██▌       | 38/150 [00:07<00:23,  4.85it/s]\u001b[A\n",
      "Evaluating workflow:  27%|██▋       | 41/150 [00:07<00:14,  7.28it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  29%|██▊       | 43/150 [00:07<00:19,  5.60it/s]\u001b[A\n",
      "Evaluating workflow:  31%|███       | 46/150 [00:07<00:12,  8.04it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  33%|███▎      | 49/150 [00:08<00:10,  9.87it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  34%|███▍      | 51/150 [00:08<00:11,  8.66it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  35%|███▌      | 53/150 [00:09<00:16,  5.74it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  37%|███▋      | 55/150 [00:09<00:17,  5.55it/s]\u001b[A\n",
      "Evaluating workflow:  37%|███▋      | 56/150 [00:09<00:15,  5.92it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  39%|███▊      | 58/150 [00:09<00:14,  6.47it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  39%|███▉      | 59/150 [00:10<00:21,  4.16it/s]\u001b[A\n",
      "Evaluating workflow:  40%|████      | 60/150 [00:10<00:19,  4.66it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  41%|████      | 61/150 [00:10<00:16,  5.27it/s]\u001b[A\n",
      "Evaluating workflow:  43%|████▎     | 64/150 [00:10<00:09,  8.78it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  44%|████▍     | 66/150 [00:11<00:10,  7.81it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  45%|████▌     | 68/150 [00:11<00:11,  6.91it/s]\u001b[A\n",
      "Evaluating workflow:  47%|████▋     | 70/150 [00:11<00:10,  7.96it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  48%|████▊     | 72/150 [00:11<00:09,  8.48it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  49%|████▉     | 74/150 [00:12<00:09,  8.04it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  51%|█████▏    | 77/150 [00:12<00:07, 10.28it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  53%|█████▎    | 79/150 [00:12<00:08,  8.33it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  53%|█████▎    | 80/150 [00:12<00:09,  7.13it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  54%|█████▍    | 81/150 [00:13<00:10,  6.34it/s]\u001b[A\n",
      "Evaluating workflow:  56%|█████▌    | 84/150 [00:13<00:06,  9.54it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  57%|█████▋    | 86/150 [00:13<00:10,  5.87it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  58%|█████▊    | 87/150 [00:14<00:12,  4.86it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  60%|██████    | 90/150 [00:14<00:10,  5.80it/s]\u001b[A\n",
      "Evaluating workflow:  61%|██████    | 91/150 [00:14<00:09,  6.24it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  62%|██████▏   | 93/150 [00:15<00:08,  7.05it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  65%|██████▍   | 97/150 [00:15<00:05,  8.95it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  67%|██████▋   | 100/150 [00:15<00:07,  7.09it/s]\u001b[A\n",
      "Evaluating workflow:  67%|██████▋   | 101/150 [00:16<00:06,  7.12it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  69%|██████▊   | 103/150 [00:16<00:05,  7.90it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  69%|██████▉   | 104/150 [00:16<00:07,  6.06it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  71%|███████▏  | 107/150 [00:16<00:06,  6.86it/s]\u001b[A\n",
      "Evaluating workflow:  73%|███████▎  | 109/150 [00:17<00:05,  8.17it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  73%|███████▎  | 110/150 [00:17<00:05,  7.91it/s]\u001b[A\n",
      "Evaluating workflow:  74%|███████▍  | 111/150 [00:17<00:05,  7.43it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  75%|███████▍  | 112/150 [00:18<00:09,  3.84it/s]\u001b[A\n",
      "Evaluating workflow:  75%|███████▌  | 113/150 [00:18<00:08,  4.31it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  77%|███████▋  | 115/150 [00:18<00:07,  4.39it/s]\u001b[A\n",
      "Evaluating workflow:  77%|███████▋  | 116/150 [00:18<00:06,  5.00it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  78%|███████▊  | 117/150 [00:18<00:05,  5.62it/s]\u001b[A\n",
      "Evaluating workflow:  79%|███████▉  | 119/150 [00:19<00:04,  7.67it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  81%|████████  | 121/150 [00:19<00:03,  8.53it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  82%|████████▏ | 123/150 [00:19<00:03,  6.93it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  84%|████████▍ | 126/150 [00:19<00:03,  7.51it/s]\u001b[A\n",
      "Evaluating workflow:  85%|████████▍ | 127/150 [00:20<00:03,  7.07it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  86%|████████▌ | 129/150 [00:20<00:02,  8.54it/s]\u001b[A\n",
      "Evaluating workflow:  88%|████████▊ | 132/150 [00:20<00:01, 11.48it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  89%|████████▉ | 134/150 [00:21<00:02,  6.80it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  91%|█████████ | 136/150 [00:21<00:02,  6.26it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  91%|█████████▏| 137/150 [00:21<00:02,  4.86it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  92%|█████████▏| 138/150 [00:22<00:02,  4.82it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  93%|█████████▎| 140/150 [00:22<00:02,  4.76it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  94%|█████████▍| 141/150 [00:22<00:01,  4.67it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  97%|█████████▋| 145/150 [00:23<00:00,  7.11it/s]\u001b[A\n",
      "Evaluating workflow:  97%|█████████▋| 146/150 [00:23<00:00,  6.97it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  98%|█████████▊| 147/150 [00:23<00:00,  5.24it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  99%|█████████▊| 148/150 [00:24<00:00,  3.30it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  99%|█████████▉| 149/150 [00:24<00:00,  2.76it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 150/150 [00:27<00:00,  5.53it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 09:09:56.348\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 12 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.5}\u001b[0m\n",
      "\u001b[32m2026-01-13 09:09:56.348\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m291\u001b[0m - \u001b[1mMetrics are worse than the best snapshot which has {'f1': 0.0, 'em': 0.0, 'acc': 0.5666666666666667}. Rolling back to the best snapshot.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 60%|██████    | 12/20 [34:58<23:53, 179.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 09:09:56.352\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2026-01-13 09:10:15.363\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2026-01-13 09:11:59.511\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2026-01-13 09:12:10.923\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2026-01-13 09:12:10.923\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 13 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/150 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   1%|          | 1/150 [00:01<03:37,  1.46s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   2%|▏         | 3/150 [00:01<01:09,  2.11it/s]\u001b[A\n",
      "Evaluating workflow:   5%|▍         | 7/150 [00:01<00:25,  5.59it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   6%|▌         | 9/150 [00:02<00:34,  4.09it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   7%|▋         | 10/150 [00:02<00:35,  3.98it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   7%|▋         | 11/150 [00:03<00:40,  3.42it/s]\u001b[A\n",
      "Evaluating workflow:   8%|▊         | 12/150 [00:03<00:34,  4.02it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   9%|▉         | 14/150 [00:03<00:23,  5.73it/s]\u001b[A\n",
      "Evaluating workflow:  13%|█▎        | 19/150 [00:03<00:11, 11.39it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  14%|█▍        | 21/150 [00:03<00:11, 11.05it/s]\u001b[A\n",
      "Evaluating workflow:  15%|█▌        | 23/150 [00:04<00:11, 11.15it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  17%|█▋        | 25/150 [00:04<00:12,  9.79it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  18%|█▊        | 27/150 [00:05<00:23,  5.25it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  20%|██        | 30/150 [00:05<00:16,  7.12it/s]\u001b[A\n",
      "Evaluating workflow:  23%|██▎       | 35/150 [00:05<00:10, 11.18it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  25%|██▍       | 37/150 [00:06<00:22,  5.02it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  26%|██▌       | 39/150 [00:07<00:22,  5.04it/s]\u001b[A\n",
      "Evaluating workflow:  27%|██▋       | 41/150 [00:07<00:17,  6.10it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  29%|██▊       | 43/150 [00:07<00:14,  7.27it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  30%|███       | 45/150 [00:07<00:17,  6.05it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  32%|███▏      | 48/150 [00:07<00:12,  8.44it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  33%|███▎      | 50/150 [00:08<00:14,  6.88it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  35%|███▍      | 52/150 [00:08<00:13,  7.03it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  36%|███▌      | 54/150 [00:09<00:14,  6.47it/s]\u001b[A\n",
      "Evaluating workflow:  37%|███▋      | 56/150 [00:09<00:12,  7.45it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  39%|███▊      | 58/150 [00:09<00:10,  8.45it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  40%|████      | 60/150 [00:09<00:12,  7.27it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  41%|████      | 61/150 [00:10<00:18,  4.83it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  42%|████▏     | 63/150 [00:11<00:22,  3.81it/s]\u001b[A\n",
      "Evaluating workflow:  43%|████▎     | 65/150 [00:11<00:16,  5.11it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  46%|████▌     | 69/150 [00:11<00:10,  8.05it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  47%|████▋     | 71/150 [00:11<00:12,  6.13it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  49%|████▉     | 74/150 [00:12<00:11,  6.53it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  51%|█████     | 76/150 [00:12<00:10,  7.04it/s]\u001b[A\n",
      "Evaluating workflow:  52%|█████▏    | 78/150 [00:12<00:08,  8.24it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  53%|█████▎    | 80/150 [00:12<00:08,  8.52it/s]\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▌    | 83/150 [00:13<00:06, 10.89it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  57%|█████▋    | 85/150 [00:13<00:09,  7.04it/s]\u001b[A\n",
      "Evaluating workflow:  58%|█████▊    | 87/150 [00:13<00:08,  7.68it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  59%|█████▉    | 89/150 [00:13<00:07,  8.23it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  61%|██████    | 91/150 [00:15<00:13,  4.23it/s]\u001b[A\n",
      "Evaluating workflow:  61%|██████▏   | 92/150 [00:15<00:12,  4.54it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  63%|██████▎   | 94/150 [00:15<00:09,  5.85it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  64%|██████▍   | 96/150 [00:15<00:09,  5.58it/s]\u001b[A\n",
      "Evaluating workflow:  66%|██████▌   | 99/150 [00:15<00:06,  8.20it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  68%|██████▊   | 102/150 [00:16<00:08,  5.92it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  69%|██████▉   | 104/150 [00:16<00:07,  5.92it/s]\u001b[A\n",
      "Evaluating workflow:  71%|███████   | 106/150 [00:17<00:06,  7.02it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  72%|███████▏  | 108/150 [00:17<00:04,  8.49it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  73%|███████▎  | 110/150 [00:17<00:04,  8.02it/s]\u001b[A\n",
      "Evaluating workflow:  75%|███████▌  | 113/150 [00:17<00:03, 10.06it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  77%|███████▋  | 115/150 [00:18<00:04,  7.60it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  78%|███████▊  | 117/150 [00:18<00:06,  5.29it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  79%|███████▊  | 118/150 [00:18<00:06,  5.08it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  79%|███████▉  | 119/150 [00:19<00:06,  4.65it/s]\u001b[A\n",
      "Evaluating workflow:  81%|████████▏ | 122/150 [00:19<00:03,  7.19it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  83%|████████▎ | 124/150 [00:19<00:03,  8.33it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  84%|████████▍ | 126/150 [00:19<00:03,  7.27it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  85%|████████▍ | 127/150 [00:20<00:03,  7.09it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  87%|████████▋ | 130/150 [00:20<00:02,  7.03it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  87%|████████▋ | 131/150 [00:20<00:02,  6.51it/s]\u001b[A\n",
      "Evaluating workflow:  88%|████████▊ | 132/150 [00:20<00:02,  6.31it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  89%|████████▊ | 133/150 [00:21<00:02,  6.53it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  90%|█████████ | 135/150 [00:21<00:02,  7.11it/s]\u001b[A\n",
      "Evaluating workflow:  91%|█████████▏| 137/150 [00:21<00:01,  9.16it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  93%|█████████▎| 139/150 [00:21<00:01, 10.36it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  94%|█████████▍| 141/150 [00:21<00:01,  7.42it/s]\u001b[A\n",
      "Evaluating workflow:  95%|█████████▍| 142/150 [00:22<00:01,  7.48it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  95%|█████████▌| 143/150 [00:22<00:01,  4.72it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  96%|█████████▌| 144/150 [00:22<00:01,  4.44it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  97%|█████████▋| 145/150 [00:23<00:01,  4.15it/s]\u001b[A\n",
      "Evaluating workflow:  97%|█████████▋| 146/150 [00:23<00:00,  4.36it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  99%|█████████▊| 148/150 [00:24<00:00,  3.54it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  99%|█████████▉| 149/150 [00:24<00:00,  3.76it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 150/150 [00:24<00:00,  6.12it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 09:12:35.547\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 13 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.5733333333333334}\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 65%|██████▌   | 13/20 [37:38<20:12, 173.14s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 09:12:35.548\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2026-01-13 09:12:53.454\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2026-01-13 09:14:39.243\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2026-01-13 09:14:55.779\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2026-01-13 09:14:55.779\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 14 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/150 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   1%|          | 1/150 [00:01<04:24,  1.78s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   4%|▍         | 6/150 [00:02<00:37,  3.85it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   5%|▌         | 8/150 [00:03<00:50,  2.80it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   7%|▋         | 10/150 [00:03<00:38,  3.61it/s]\u001b[A\n",
      "Evaluating workflow:   8%|▊         | 12/150 [00:03<00:28,  4.77it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   9%|▉         | 14/150 [00:03<00:26,  5.07it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  11%|█         | 16/150 [00:03<00:21,  6.31it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  12%|█▏        | 18/150 [00:04<00:18,  7.29it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  13%|█▎        | 20/150 [00:04<00:17,  7.22it/s]\u001b[A\n",
      "Evaluating workflow:  15%|█▍        | 22/150 [00:04<00:16,  7.93it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  15%|█▌        | 23/150 [00:04<00:17,  7.31it/s]\u001b[A\n",
      "Evaluating workflow:  17%|█▋        | 25/150 [00:04<00:13,  9.16it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  18%|█▊        | 27/150 [00:05<00:21,  5.84it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  20%|██        | 30/150 [00:05<00:14,  8.43it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  21%|██▏       | 32/150 [00:06<00:23,  4.98it/s]\u001b[A\n",
      "Evaluating workflow:  23%|██▎       | 34/150 [00:06<00:18,  6.25it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  25%|██▍       | 37/150 [00:06<00:13,  8.08it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  26%|██▌       | 39/150 [00:07<00:23,  4.78it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  27%|██▋       | 41/150 [00:07<00:21,  5.11it/s]\u001b[A\n",
      "Evaluating workflow:  28%|██▊       | 42/150 [00:08<00:20,  5.20it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  29%|██▉       | 44/150 [00:08<00:19,  5.55it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  31%|███       | 46/150 [00:08<00:16,  6.23it/s]\u001b[A\n",
      "Evaluating workflow:  31%|███▏      | 47/150 [00:08<00:16,  6.17it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  33%|███▎      | 49/150 [00:09<00:15,  6.62it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  33%|███▎      | 50/150 [00:09<00:15,  6.32it/s]\u001b[A\n",
      "Evaluating workflow:  35%|███▍      | 52/150 [00:09<00:11,  8.21it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  36%|███▌      | 54/150 [00:09<00:09,  9.62it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  37%|███▋      | 56/150 [00:09<00:10,  8.93it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  39%|███▊      | 58/150 [00:10<00:10,  9.01it/s]\u001b[A\n",
      "Evaluating workflow:  39%|███▉      | 59/150 [00:10<00:11,  7.95it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  40%|████      | 60/150 [00:10<00:15,  5.83it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  41%|████▏     | 62/150 [00:10<00:14,  6.20it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  42%|████▏     | 63/150 [00:11<00:20,  4.26it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  44%|████▍     | 66/150 [00:12<00:19,  4.24it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  45%|████▌     | 68/150 [00:12<00:17,  4.59it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  47%|████▋     | 70/150 [00:12<00:15,  5.12it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  48%|████▊     | 72/150 [00:13<00:14,  5.57it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  49%|████▉     | 74/150 [00:13<00:12,  6.33it/s]\u001b[A\n",
      "Evaluating workflow:  51%|█████     | 76/150 [00:13<00:09,  7.78it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  53%|█████▎    | 79/150 [00:13<00:08,  8.56it/s]\u001b[A\n",
      "Evaluating workflow:  54%|█████▍    | 81/150 [00:13<00:07,  9.64it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  55%|█████▌    | 83/150 [00:13<00:06,  9.83it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  57%|█████▋    | 85/150 [00:14<00:07,  8.66it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  57%|█████▋    | 86/150 [00:14<00:12,  5.29it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  59%|█████▊    | 88/150 [00:15<00:11,  5.51it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  59%|█████▉    | 89/150 [00:15<00:11,  5.39it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  60%|██████    | 90/150 [00:15<00:13,  4.46it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  61%|██████    | 91/150 [00:15<00:13,  4.35it/s]\u001b[A\n",
      "Evaluating workflow:  63%|██████▎   | 94/150 [00:16<00:07,  7.29it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  64%|██████▍   | 96/150 [00:16<00:08,  6.46it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  65%|██████▍   | 97/150 [00:16<00:09,  5.60it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  65%|██████▌   | 98/150 [00:17<00:14,  3.69it/s]\u001b[A\n",
      "Evaluating workflow:  68%|██████▊   | 102/150 [00:17<00:06,  7.32it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  69%|██████▉   | 104/150 [00:17<00:05,  8.32it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  71%|███████   | 106/150 [00:17<00:05,  7.44it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  72%|███████▏  | 108/150 [00:18<00:06,  6.29it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  73%|███████▎  | 110/150 [00:18<00:06,  6.19it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  75%|███████▍  | 112/150 [00:19<00:06,  5.72it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  76%|███████▌  | 114/150 [00:19<00:05,  6.09it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  77%|███████▋  | 116/150 [00:19<00:05,  6.40it/s]\u001b[A\n",
      "Evaluating workflow:  78%|███████▊  | 117/150 [00:19<00:05,  6.13it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  79%|███████▉  | 119/150 [00:20<00:04,  7.35it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  81%|████████  | 121/150 [00:20<00:04,  6.84it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  81%|████████▏ | 122/150 [00:20<00:05,  4.88it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  82%|████████▏ | 123/150 [00:21<00:05,  4.64it/s]\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 126/150 [00:21<00:03,  6.91it/s]\u001b[A\n",
      "Evaluating workflow:  85%|████████▍ | 127/150 [00:21<00:03,  6.39it/s]\u001b[A\n",
      "Evaluating workflow:  85%|████████▌ | 128/150 [00:21<00:03,  6.56it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  86%|████████▌ | 129/150 [00:21<00:03,  6.18it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  87%|████████▋ | 130/150 [00:22<00:03,  5.08it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  87%|████████▋ | 131/150 [00:22<00:04,  4.64it/s]\u001b[A\n",
      "Evaluating workflow:  88%|████████▊ | 132/150 [00:22<00:03,  4.88it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  89%|████████▊ | 133/150 [00:23<00:04,  3.77it/s]\u001b[A\n",
      "Evaluating workflow:  89%|████████▉ | 134/150 [00:23<00:03,  4.47it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  90%|█████████ | 135/150 [00:23<00:02,  5.24it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  91%|█████████▏| 137/150 [00:23<00:02,  5.86it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  93%|█████████▎| 139/150 [00:23<00:01,  6.08it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  94%|█████████▍| 141/150 [00:24<00:01,  5.99it/s]\u001b[A\n",
      "Evaluating workflow:  95%|█████████▍| 142/150 [00:24<00:01,  6.44it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  95%|█████████▌| 143/150 [00:24<00:01,  4.78it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  97%|█████████▋| 145/150 [00:24<00:00,  5.65it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  97%|█████████▋| 146/150 [00:25<00:00,  5.00it/s]\u001b[A\n",
      "Evaluating workflow:  98%|█████████▊| 147/150 [00:25<00:00,  5.03it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  99%|█████████▉| 149/150 [00:25<00:00,  6.56it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 150/150 [00:26<00:00,  5.76it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 09:15:21.931\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 14 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.5133333333333333}\u001b[0m\n",
      "\u001b[32m2026-01-13 09:15:21.931\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m291\u001b[0m - \u001b[1mMetrics are worse than the best snapshot which has {'f1': 0.0, 'em': 0.0, 'acc': 0.5733333333333334}. Rolling back to the best snapshot.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 70%|███████   | 14/20 [40:24<17:06, 171.10s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 09:15:21.935\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2026-01-13 09:15:44.969\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2026-01-13 09:17:48.474\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2026-01-13 09:18:05.282\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2026-01-13 09:18:05.282\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 15 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/150 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   1%|          | 1/150 [00:01<04:14,  1.71s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   2%|▏         | 3/150 [00:02<01:30,  1.62it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   3%|▎         | 4/150 [00:02<01:24,  1.74it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   3%|▎         | 5/150 [00:03<01:24,  1.71it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   4%|▍         | 6/150 [00:03<01:08,  2.12it/s]\u001b[A\n",
      "Evaluating workflow:   5%|▍         | 7/150 [00:03<00:51,  2.77it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   7%|▋         | 10/150 [00:03<00:25,  5.50it/s]\u001b[A\n",
      "Evaluating workflow:   7%|▋         | 11/150 [00:03<00:25,  5.54it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   8%|▊         | 12/150 [00:04<00:25,  5.38it/s]\u001b[A\n",
      "Evaluating workflow:   9%|▊         | 13/150 [00:04<00:25,  5.36it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  12%|█▏        | 18/150 [00:04<00:14,  8.88it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  14%|█▍        | 21/150 [00:05<00:17,  7.27it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  15%|█▌        | 23/150 [00:05<00:21,  6.02it/s]\u001b[A\n",
      "Evaluating workflow:  16%|█▌        | 24/150 [00:05<00:21,  5.87it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  17%|█▋        | 25/150 [00:06<00:20,  6.18it/s]\u001b[A\n",
      "Evaluating workflow:  17%|█▋        | 26/150 [00:06<00:20,  5.97it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  19%|█▉        | 29/150 [00:06<00:20,  5.95it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  20%|██        | 30/150 [00:07<00:33,  3.59it/s]\u001b[A\n",
      "Evaluating workflow:  21%|██        | 31/150 [00:07<00:29,  4.06it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  22%|██▏       | 33/150 [00:08<00:25,  4.55it/s]\u001b[A\n",
      "Evaluating workflow:  23%|██▎       | 35/150 [00:08<00:19,  5.97it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  25%|██▌       | 38/150 [00:08<00:13,  8.16it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  27%|██▋       | 40/150 [00:08<00:13,  8.18it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  27%|██▋       | 41/150 [00:09<00:18,  5.78it/s]\u001b[A\n",
      "Evaluating workflow:  28%|██▊       | 42/150 [00:09<00:17,  6.22it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  30%|███       | 45/150 [00:09<00:11,  9.49it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  31%|███▏      | 47/150 [00:09<00:18,  5.46it/s]\u001b[A\n",
      "Evaluating workflow:  33%|███▎      | 49/150 [00:10<00:15,  6.72it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  34%|███▍      | 51/150 [00:10<00:12,  8.02it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  35%|███▌      | 53/150 [00:10<00:12,  7.69it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  37%|███▋      | 55/150 [00:11<00:23,  4.08it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  39%|███▊      | 58/150 [00:11<00:16,  5.45it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  41%|████      | 61/150 [00:12<00:13,  6.39it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  42%|████▏     | 63/150 [00:12<00:13,  6.31it/s]\u001b[A\n",
      "Evaluating workflow:  43%|████▎     | 64/150 [00:12<00:13,  6.35it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  43%|████▎     | 65/150 [00:13<00:17,  4.86it/s]\u001b[A\n",
      "Evaluating workflow:  45%|████▍     | 67/150 [00:13<00:12,  6.46it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  45%|████▌     | 68/150 [00:13<00:16,  5.00it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  47%|████▋     | 70/150 [00:13<00:15,  5.08it/s]\u001b[A\n",
      "Evaluating workflow:  48%|████▊     | 72/150 [00:14<00:11,  6.79it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  49%|████▉     | 74/150 [00:14<00:09,  7.60it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  51%|█████     | 76/150 [00:14<00:13,  5.46it/s]\u001b[A\n",
      "Evaluating workflow:  51%|█████▏    | 77/150 [00:14<00:12,  5.92it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  52%|█████▏    | 78/150 [00:15<00:11,  6.05it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  53%|█████▎    | 79/150 [00:15<00:18,  3.76it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  55%|█████▍    | 82/150 [00:16<00:14,  4.81it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  57%|█████▋    | 86/150 [00:16<00:11,  5.65it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  59%|█████▊    | 88/150 [00:17<00:12,  5.14it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  59%|█████▉    | 89/150 [00:17<00:13,  4.58it/s]\u001b[A\n",
      "Evaluating workflow:  61%|██████    | 91/150 [00:17<00:10,  5.88it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  61%|██████▏   | 92/150 [00:17<00:09,  6.14it/s]\u001b[A\n",
      "Evaluating workflow:  63%|██████▎   | 94/150 [00:17<00:07,  7.15it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  63%|██████▎   | 95/150 [00:18<00:08,  6.53it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  65%|██████▍   | 97/150 [00:18<00:09,  5.77it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  66%|██████▌   | 99/150 [00:19<00:09,  5.13it/s]\u001b[A\n",
      "Evaluating workflow:  67%|██████▋   | 100/150 [00:19<00:09,  5.46it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  67%|██████▋   | 101/150 [00:19<00:08,  5.91it/s]\u001b[A\n",
      "Evaluating workflow:  69%|██████▊   | 103/150 [00:19<00:06,  7.18it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  70%|███████   | 105/150 [00:20<00:09,  4.55it/s]\u001b[A\n",
      "Evaluating workflow:  71%|███████   | 106/150 [00:20<00:08,  5.00it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  73%|███████▎  | 109/150 [00:20<00:05,  7.98it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  74%|███████▍  | 111/150 [00:20<00:05,  6.66it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  75%|███████▌  | 113/150 [00:21<00:07,  4.96it/s]\u001b[A\n",
      "Evaluating workflow:  76%|███████▌  | 114/150 [00:21<00:06,  5.29it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  77%|███████▋  | 115/150 [00:22<00:08,  4.15it/s]\u001b[A\n",
      "Evaluating workflow:  79%|███████▊  | 118/150 [00:22<00:04,  6.59it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  80%|████████  | 120/150 [00:22<00:04,  6.15it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  81%|████████  | 121/150 [00:22<00:05,  5.76it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  82%|████████▏ | 123/150 [00:23<00:05,  5.02it/s]\u001b[A\n",
      "Evaluating workflow:  83%|████████▎ | 125/150 [00:23<00:03,  6.34it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  84%|████████▍ | 126/150 [00:23<00:03,  6.74it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  85%|████████▍ | 127/150 [00:23<00:03,  6.17it/s]\u001b[A\n",
      "Evaluating workflow:  86%|████████▌ | 129/150 [00:23<00:02,  7.78it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  87%|████████▋ | 130/150 [00:24<00:03,  5.77it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  88%|████████▊ | 132/150 [00:24<00:02,  6.79it/s]\u001b[A\n",
      "Evaluating workflow:  89%|████████▊ | 133/150 [00:24<00:02,  6.60it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  89%|████████▉ | 134/150 [00:25<00:03,  5.14it/s]\u001b[A\n",
      "Evaluating workflow:  91%|█████████ | 136/150 [00:25<00:02,  6.81it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  92%|█████████▏| 138/150 [00:26<00:03,  3.95it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  93%|█████████▎| 140/150 [00:26<00:02,  3.48it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  94%|█████████▍| 141/150 [00:27<00:02,  3.19it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  95%|█████████▍| 142/150 [00:27<00:02,  3.39it/s]\u001b[A\n",
      "Evaluating workflow:  95%|█████████▌| 143/150 [00:27<00:01,  3.89it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  97%|█████████▋| 145/150 [00:27<00:00,  5.45it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  97%|█████████▋| 146/150 [00:27<00:00,  5.57it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  98%|█████████▊| 147/150 [00:28<00:00,  5.04it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  99%|█████████▊| 148/150 [00:29<00:01,  2.00it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  99%|█████████▉| 149/150 [00:29<00:00,  2.36it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 150/150 [00:30<00:00,  4.94it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 09:18:35.784\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 15 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.47333333333333333}\u001b[0m\n",
      "\u001b[32m2026-01-13 09:18:35.784\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m291\u001b[0m - \u001b[1mMetrics are worse than the best snapshot which has {'f1': 0.0, 'em': 0.0, 'acc': 0.5733333333333334}. Rolling back to the best snapshot.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 75%|███████▌  | 15/20 [43:38<14:49, 177.96s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 09:18:35.788\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2026-01-13 09:18:55.442\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2026-01-13 09:20:36.211\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2026-01-13 09:20:49.108\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2026-01-13 09:20:49.109\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 16 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/150 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   1%|          | 1/150 [00:01<04:22,  1.76s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   3%|▎         | 4/150 [00:02<01:16,  1.91it/s]\u001b[A\n",
      "Evaluating workflow:   3%|▎         | 5/150 [00:02<01:01,  2.37it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   4%|▍         | 6/150 [00:02<00:50,  2.85it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   5%|▍         | 7/150 [00:03<00:44,  3.22it/s]\u001b[A\n",
      "Evaluating workflow:   6%|▌         | 9/150 [00:03<00:28,  4.91it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   7%|▋         | 11/150 [00:03<00:21,  6.50it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   8%|▊         | 12/150 [00:03<00:22,  6.07it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   9%|▉         | 14/150 [00:04<00:29,  4.55it/s]\u001b[A\n",
      "Evaluating workflow:  13%|█▎        | 19/150 [00:04<00:13,  9.49it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  14%|█▍        | 21/150 [00:04<00:12, 10.14it/s]\u001b[A\n",
      "Evaluating workflow:  15%|█▌        | 23/150 [00:04<00:11, 10.77it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  17%|█▋        | 25/150 [00:05<00:20,  6.10it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  18%|█▊        | 27/150 [00:06<00:29,  4.21it/s]\u001b[A\n",
      "Evaluating workflow:  19%|█▊        | 28/150 [00:06<00:28,  4.35it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  19%|█▉        | 29/150 [00:06<00:32,  3.77it/s]\u001b[A\n",
      "Evaluating workflow:  20%|██        | 30/150 [00:06<00:29,  4.03it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  22%|██▏       | 33/150 [00:07<00:19,  5.90it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  23%|██▎       | 34/150 [00:07<00:19,  6.05it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  23%|██▎       | 35/150 [00:07<00:22,  5.04it/s]\u001b[A\n",
      "Evaluating workflow:  25%|██▍       | 37/150 [00:07<00:16,  6.66it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  27%|██▋       | 40/150 [00:07<00:11,  9.85it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  28%|██▊       | 42/150 [00:08<00:12,  8.90it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  29%|██▉       | 44/150 [00:08<00:10,  9.91it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  31%|███       | 46/150 [00:08<00:14,  7.28it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  31%|███▏      | 47/150 [00:09<00:18,  5.52it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  32%|███▏      | 48/150 [00:09<00:19,  5.30it/s]\u001b[A\n",
      "Evaluating workflow:  33%|███▎      | 49/150 [00:09<00:17,  5.76it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  34%|███▍      | 51/150 [00:09<00:13,  7.54it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  35%|███▍      | 52/150 [00:10<00:19,  4.99it/s]\u001b[A\n",
      "Evaluating workflow:  35%|███▌      | 53/150 [00:10<00:18,  5.15it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  37%|███▋      | 55/150 [00:10<00:14,  6.39it/s]\u001b[A\n",
      "Evaluating workflow:  37%|███▋      | 56/150 [00:10<00:15,  6.22it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  38%|███▊      | 57/150 [00:10<00:14,  6.50it/s]\u001b[A\n",
      "Evaluating workflow:  39%|███▉      | 59/150 [00:10<00:10,  8.76it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  41%|████      | 61/150 [00:10<00:08, 10.93it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  42%|████▏     | 63/150 [00:11<00:10,  8.34it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  43%|████▎     | 65/150 [00:11<00:13,  6.28it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  44%|████▍     | 66/150 [00:12<00:15,  5.52it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  45%|████▍     | 67/150 [00:12<00:16,  5.00it/s]\u001b[A\n",
      "Evaluating workflow:  46%|████▌     | 69/150 [00:12<00:12,  6.57it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  47%|████▋     | 71/150 [00:12<00:13,  5.82it/s]\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 72/150 [00:13<00:13,  5.63it/s]\u001b[A\n",
      "Evaluating workflow:  49%|████▉     | 74/150 [00:13<00:10,  7.27it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  50%|█████     | 75/150 [00:13<00:11,  6.53it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  51%|█████▏    | 77/150 [00:13<00:10,  7.02it/s]\u001b[A\n",
      "Evaluating workflow:  53%|█████▎    | 79/150 [00:13<00:09,  7.88it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  53%|█████▎    | 80/150 [00:14<00:11,  5.90it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  54%|█████▍    | 81/150 [00:14<00:18,  3.79it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  56%|█████▌    | 84/150 [00:15<00:11,  5.77it/s]\u001b[A\n",
      "Evaluating workflow:  57%|█████▋    | 86/150 [00:15<00:09,  6.99it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  59%|█████▊    | 88/150 [00:15<00:07,  8.27it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  60%|██████    | 90/150 [00:15<00:09,  6.42it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  61%|██████    | 91/150 [00:16<00:09,  5.96it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  61%|██████▏   | 92/150 [00:16<00:14,  4.09it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  62%|██████▏   | 93/150 [00:16<00:13,  4.12it/s]\u001b[A\n",
      "Evaluating workflow:  63%|██████▎   | 94/150 [00:16<00:11,  4.68it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  65%|██████▍   | 97/150 [00:17<00:12,  4.21it/s]\u001b[A\n",
      "Evaluating workflow:  66%|██████▌   | 99/150 [00:17<00:09,  5.41it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  67%|██████▋   | 100/150 [00:18<00:11,  4.26it/s]\u001b[A\n",
      "Evaluating workflow:  69%|██████▊   | 103/150 [00:18<00:06,  6.84it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  70%|███████   | 105/150 [00:18<00:07,  6.20it/s]\u001b[A\n",
      "Evaluating workflow:  71%|███████▏  | 107/150 [00:18<00:05,  7.64it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  73%|███████▎  | 109/150 [00:19<00:06,  6.63it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  73%|███████▎  | 110/150 [00:19<00:07,  5.10it/s]\u001b[A\n",
      "Evaluating workflow:  75%|███████▍  | 112/150 [00:19<00:06,  6.21it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  75%|███████▌  | 113/150 [00:20<00:06,  5.98it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  77%|███████▋  | 115/150 [00:20<00:05,  6.22it/s]\u001b[A\n",
      "Evaluating workflow:  77%|███████▋  | 116/150 [00:20<00:05,  6.27it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  78%|███████▊  | 117/150 [00:20<00:05,  5.57it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  79%|███████▉  | 119/150 [00:21<00:04,  6.33it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  81%|████████▏ | 122/150 [00:22<00:06,  4.37it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  82%|████████▏ | 123/150 [00:22<00:06,  4.29it/s]\u001b[A\n",
      "Evaluating workflow:  83%|████████▎ | 125/150 [00:22<00:04,  5.78it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  87%|████████▋ | 130/150 [00:22<00:02,  8.81it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  88%|████████▊ | 132/150 [00:23<00:02,  6.38it/s]\u001b[A\n",
      "Evaluating workflow:  89%|████████▊ | 133/150 [00:23<00:02,  6.21it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  91%|█████████ | 136/150 [00:23<00:01,  7.87it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  91%|█████████▏| 137/150 [00:23<00:01,  7.84it/s]\u001b[A\n",
      "Evaluating workflow:  92%|█████████▏| 138/150 [00:23<00:01,  7.91it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  93%|█████████▎| 140/150 [00:24<00:01,  5.34it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  95%|█████████▍| 142/150 [00:25<00:01,  4.17it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  96%|█████████▌| 144/150 [00:25<00:01,  4.88it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  97%|█████████▋| 145/150 [00:25<00:01,  4.29it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  97%|█████████▋| 146/150 [00:26<00:00,  4.42it/s]\u001b[A\n",
      "Evaluating workflow:  98%|█████████▊| 147/150 [00:26<00:00,  3.89it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  99%|█████████▊| 148/150 [00:26<00:00,  4.04it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  99%|█████████▉| 149/150 [00:27<00:00,  3.31it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 150/150 [00:28<00:00,  5.20it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 09:21:18.068\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 16 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.48}\u001b[0m\n",
      "\u001b[32m2026-01-13 09:21:18.068\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m291\u001b[0m - \u001b[1mMetrics are worse than the best snapshot which has {'f1': 0.0, 'em': 0.0, 'acc': 0.5733333333333334}. Rolling back to the best snapshot.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 80%|████████  | 16/20 [46:20<11:32, 173.24s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 09:21:18.071\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2026-01-13 09:21:35.772\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2026-01-13 09:23:26.215\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2026-01-13 09:23:37.637\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2026-01-13 09:23:37.637\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 17 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/150 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   1%|          | 1/150 [00:01<04:05,  1.65s/it]\u001b[A\n",
      "Evaluating workflow:   2%|▏         | 3/150 [00:01<01:08,  2.15it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   3%|▎         | 5/150 [00:01<00:39,  3.65it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   5%|▌         | 8/150 [00:03<00:49,  2.88it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   9%|▉         | 14/150 [00:03<00:21,  6.21it/s]\u001b[A\n",
      "Evaluating workflow:  11%|█▏        | 17/150 [00:03<00:16,  7.95it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  15%|█▍        | 22/150 [00:03<00:10, 12.02it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  17%|█▋        | 25/150 [00:04<00:17,  7.23it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  18%|█▊        | 27/150 [00:04<00:16,  7.26it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  19%|█▉        | 29/150 [00:05<00:18,  6.70it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  21%|██        | 31/150 [00:05<00:17,  6.81it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  22%|██▏       | 33/150 [00:05<00:17,  6.55it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  23%|██▎       | 34/150 [00:06<00:19,  6.07it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  23%|██▎       | 35/150 [00:06<00:24,  4.68it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  24%|██▍       | 36/150 [00:06<00:25,  4.43it/s]\u001b[A\n",
      "Evaluating workflow:  25%|██▌       | 38/150 [00:06<00:18,  6.08it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  26%|██▌       | 39/150 [00:07<00:18,  6.06it/s]\u001b[A\n",
      "Evaluating workflow:  27%|██▋       | 41/150 [00:07<00:14,  7.76it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  29%|██▊       | 43/150 [00:07<00:15,  7.09it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  30%|███       | 45/150 [00:07<00:13,  7.52it/s]\u001b[A\n",
      "Evaluating workflow:  31%|███       | 46/150 [00:07<00:14,  7.37it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  34%|███▍      | 51/150 [00:07<00:07, 13.83it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  35%|███▌      | 53/150 [00:08<00:11,  8.43it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  37%|███▋      | 55/150 [00:09<00:15,  6.04it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  38%|███▊      | 57/150 [00:09<00:14,  6.46it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  39%|███▊      | 58/150 [00:09<00:16,  5.49it/s]\u001b[A\n",
      "Evaluating workflow:  40%|████      | 60/150 [00:09<00:12,  6.96it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  41%|████▏     | 62/150 [00:10<00:11,  7.70it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  43%|████▎     | 64/150 [00:10<00:18,  4.67it/s]\u001b[A\n",
      "Evaluating workflow:  43%|████▎     | 65/150 [00:10<00:16,  5.08it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  45%|████▌     | 68/150 [00:11<00:12,  6.37it/s]\u001b[A\n",
      "Evaluating workflow:  47%|████▋     | 70/150 [00:11<00:10,  7.93it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  49%|████▊     | 73/150 [00:11<00:07, 10.19it/s]\u001b[A\n",
      "Evaluating workflow:  50%|█████     | 75/150 [00:11<00:06, 11.18it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  51%|█████▏    | 77/150 [00:12<00:09,  7.58it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  53%|█████▎    | 79/150 [00:12<00:14,  4.85it/s]\u001b[A\n",
      "Evaluating workflow:  55%|█████▍    | 82/150 [00:13<00:10,  6.66it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  56%|█████▌    | 84/150 [00:13<00:08,  7.52it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  58%|█████▊    | 87/150 [00:13<00:07,  8.62it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  59%|█████▉    | 89/150 [00:13<00:07,  7.84it/s]\u001b[A\n",
      "Evaluating workflow:  61%|██████    | 91/150 [00:13<00:06,  9.20it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  62%|██████▏   | 93/150 [00:14<00:10,  5.36it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  63%|██████▎   | 94/150 [00:14<00:10,  5.22it/s]\u001b[A\n",
      "Evaluating workflow:  65%|██████▍   | 97/150 [00:15<00:07,  7.44it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  66%|██████▌   | 99/150 [00:15<00:06,  7.95it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  68%|██████▊   | 102/150 [00:15<00:05,  9.22it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  69%|██████▉   | 104/150 [00:16<00:06,  6.66it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  70%|███████   | 105/150 [00:16<00:10,  4.36it/s]\u001b[A\n",
      "Evaluating workflow:  73%|███████▎  | 109/150 [00:16<00:05,  7.10it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  75%|███████▍  | 112/150 [00:17<00:04,  9.42it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  77%|███████▋  | 115/150 [00:17<00:03, 11.45it/s]\u001b[A\n",
      "Evaluating workflow:  78%|███████▊  | 117/150 [00:17<00:02, 12.01it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  79%|███████▉  | 119/150 [00:17<00:04,  6.89it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  81%|████████  | 121/150 [00:18<00:05,  5.47it/s]\u001b[A\n",
      "Evaluating workflow:  81%|████████▏ | 122/150 [00:18<00:04,  5.71it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  83%|████████▎ | 124/150 [00:18<00:03,  7.16it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  84%|████████▍ | 126/150 [00:18<00:02,  8.08it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  85%|████████▌ | 128/150 [00:19<00:02,  7.53it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  87%|████████▋ | 130/150 [00:19<00:03,  6.04it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  87%|████████▋ | 131/150 [00:20<00:03,  5.19it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  88%|████████▊ | 132/150 [00:20<00:03,  4.73it/s]\u001b[A\n",
      "Evaluating workflow:  89%|████████▉ | 134/150 [00:20<00:02,  6.28it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  91%|█████████ | 136/150 [00:20<00:02,  5.88it/s]\u001b[A\n",
      "Evaluating workflow:  91%|█████████▏| 137/150 [00:21<00:02,  5.84it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  92%|█████████▏| 138/150 [00:21<00:02,  5.19it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  95%|█████████▍| 142/150 [00:21<00:01,  7.47it/s]\u001b[A\n",
      "Evaluating workflow:  97%|█████████▋| 145/150 [00:21<00:00,  9.96it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  98%|█████████▊| 147/150 [00:22<00:00,  9.99it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  99%|█████████▉| 149/150 [00:23<00:00,  4.04it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 150/150 [00:23<00:00,  6.33it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 09:24:01.450\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 17 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.58}\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 85%|████████▌ | 17/20 [49:03<08:30, 170.28s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 09:24:01.451\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2026-01-13 09:24:20.781\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2026-01-13 09:26:07.311\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2026-01-13 09:26:20.455\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2026-01-13 09:26:20.455\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 18 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/150 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   1%|          | 1/150 [00:01<04:37,  1.86s/it]\u001b[A\n",
      "Evaluating workflow:   3%|▎         | 5/150 [00:02<00:44,  3.23it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   5%|▍         | 7/150 [00:02<00:34,  4.12it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   6%|▌         | 9/150 [00:03<00:45,  3.10it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   7%|▋         | 10/150 [00:03<00:52,  2.69it/s]\u001b[A\n",
      "Evaluating workflow:   9%|▊         | 13/150 [00:03<00:30,  4.46it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  11%|█         | 16/150 [00:04<00:21,  6.20it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  12%|█▏        | 18/150 [00:04<00:19,  6.89it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  14%|█▍        | 21/150 [00:04<00:14,  8.81it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  15%|█▌        | 23/150 [00:05<00:18,  6.78it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  17%|█▋        | 25/150 [00:05<00:23,  5.28it/s]\u001b[A\n",
      "Evaluating workflow:  18%|█▊        | 27/150 [00:05<00:18,  6.61it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  19%|█▉        | 29/150 [00:06<00:18,  6.39it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  21%|██        | 31/150 [00:06<00:17,  6.67it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  22%|██▏       | 33/150 [00:07<00:25,  4.60it/s]\u001b[A\n",
      "Evaluating workflow:  23%|██▎       | 34/150 [00:07<00:23,  4.91it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  24%|██▍       | 36/150 [00:07<00:20,  5.57it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  25%|██▌       | 38/150 [00:07<00:16,  6.72it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  27%|██▋       | 40/150 [00:07<00:16,  6.62it/s]\u001b[A\n",
      "Evaluating workflow:  29%|██▉       | 44/150 [00:08<00:10,  9.75it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  31%|███       | 46/150 [00:08<00:16,  6.36it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  32%|███▏      | 48/150 [00:09<00:17,  5.72it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  33%|███▎      | 49/150 [00:09<00:20,  5.03it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  33%|███▎      | 50/150 [00:09<00:21,  4.69it/s]\u001b[A\n",
      "Evaluating workflow:  34%|███▍      | 51/150 [00:09<00:18,  5.26it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  35%|███▌      | 53/150 [00:10<00:18,  5.25it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 55/150 [00:11<00:16,  5.61it/s]\u001b[A\n",
      "Evaluating workflow:  37%|███▋      | 56/150 [00:11<00:20,  4.49it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  38%|███▊      | 57/150 [00:11<00:24,  3.77it/s]\u001b[A\n",
      "Evaluating workflow:  39%|███▉      | 59/150 [00:11<00:16,  5.50it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  41%|████▏     | 62/150 [00:11<00:11,  7.60it/s]\u001b[A\n",
      "Evaluating workflow:  43%|████▎     | 64/150 [00:11<00:09,  9.28it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  45%|████▍     | 67/150 [00:12<00:08,  9.53it/s]\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 69/150 [00:12<00:08,  9.64it/s]\u001b[A\n",
      "Evaluating workflow:  47%|████▋     | 71/150 [00:12<00:07, 10.27it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  49%|████▊     | 73/150 [00:13<00:11,  6.77it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  49%|████▉     | 74/150 [00:13<00:14,  5.23it/s]\u001b[A\n",
      "Evaluating workflow:  51%|█████     | 76/150 [00:13<00:10,  6.84it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  52%|█████▏    | 78/150 [00:13<00:08,  8.20it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  53%|█████▎    | 80/150 [00:14<00:10,  6.96it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  55%|█████▍    | 82/150 [00:14<00:08,  7.57it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  55%|█████▌    | 83/150 [00:14<00:11,  5.95it/s]\u001b[A\n",
      "Evaluating workflow:  56%|█████▌    | 84/150 [00:14<00:10,  6.02it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  57%|█████▋    | 85/150 [00:15<00:12,  5.24it/s]\u001b[A\n",
      "Evaluating workflow:  57%|█████▋    | 86/150 [00:15<00:11,  5.71it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  59%|█████▊    | 88/150 [00:15<00:07,  7.78it/s]\u001b[A\n",
      "Evaluating workflow:  59%|█████▉    | 89/150 [00:15<00:08,  7.61it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  60%|██████    | 90/150 [00:15<00:09,  6.13it/s]\u001b[A\n",
      "Evaluating workflow:  61%|██████▏   | 92/150 [00:15<00:07,  7.86it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  63%|██████▎   | 94/150 [00:16<00:06,  8.96it/s]\u001b[A\n",
      "Evaluating workflow:  64%|██████▍   | 96/150 [00:16<00:05,  9.73it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  67%|██████▋   | 100/150 [00:16<00:06,  7.37it/s]\u001b[A\n",
      "Evaluating workflow:  67%|██████▋   | 101/150 [00:17<00:06,  7.64it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  68%|██████▊   | 102/150 [00:17<00:06,  7.90it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  69%|██████▊   | 103/150 [00:17<00:10,  4.68it/s]\u001b[A\n",
      "Evaluating workflow:  69%|██████▉   | 104/150 [00:17<00:08,  5.18it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  71%|███████   | 106/150 [00:17<00:06,  6.79it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  71%|███████▏  | 107/150 [00:18<00:07,  6.09it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  73%|███████▎  | 109/150 [00:18<00:05,  7.01it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  74%|███████▍  | 111/150 [00:18<00:07,  5.27it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  75%|███████▌  | 113/150 [00:19<00:06,  6.12it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  77%|███████▋  | 116/150 [00:19<00:04,  8.08it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  80%|████████  | 120/150 [00:19<00:03,  9.19it/s]\u001b[A\n",
      "Evaluating workflow:  81%|████████▏ | 122/150 [00:19<00:02, 10.00it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  83%|████████▎ | 125/150 [00:20<00:03,  7.24it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  85%|████████▍ | 127/150 [00:20<00:03,  6.33it/s]\u001b[A\n",
      "Evaluating workflow:  86%|████████▌ | 129/150 [00:21<00:02,  7.44it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  87%|████████▋ | 130/150 [00:21<00:02,  6.84it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  88%|████████▊ | 132/150 [00:21<00:02,  6.67it/s]\u001b[A\n",
      "Evaluating workflow:  89%|████████▊ | 133/150 [00:21<00:02,  6.88it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  89%|████████▉ | 134/150 [00:21<00:02,  7.08it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  90%|█████████ | 135/150 [00:22<00:03,  4.17it/s]\u001b[A\n",
      "Evaluating workflow:  91%|█████████ | 136/150 [00:22<00:03,  4.55it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  92%|█████████▏| 138/150 [00:22<00:01,  6.02it/s]\u001b[A\n",
      "Evaluating workflow:  93%|█████████▎| 140/150 [00:22<00:01,  7.96it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  95%|█████████▍| 142/150 [00:23<00:00,  8.60it/s]\u001b[A\n",
      "Evaluating workflow:  96%|█████████▌| 144/150 [00:23<00:00,  9.97it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  97%|█████████▋| 146/150 [00:23<00:00, 11.06it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  99%|█████████▊| 148/150 [00:24<00:00,  6.10it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 150/150 [00:24<00:00,  6.19it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2026-01-13 09:26:44.806\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 18 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.5266666666666666}\u001b[0m\n",
      "\u001b[32m2026-01-13 09:26:44.806\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m291\u001b[0m - \u001b[1mMetrics are worse than the best snapshot which has {'f1': 0.0, 'em': 0.0, 'acc': 0.58}. Rolling back to the best snapshot.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 90%|█████████ | 18/20 [51:47<05:36, 168.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 09:26:44.810\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2026-01-13 09:27:06.749\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2026-01-13 09:29:00.851\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2026-01-13 09:29:12.468\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2026-01-13 09:29:12.468\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 19 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/150 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   1%|          | 1/150 [00:01<03:15,  1.31s/it]\u001b[A\n",
      "Evaluating workflow:   1%|▏         | 2/150 [00:01<01:32,  1.60it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   3%|▎         | 5/150 [00:01<00:29,  4.85it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   5%|▍         | 7/150 [00:01<00:23,  6.02it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   6%|▌         | 9/150 [00:02<00:29,  4.73it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   7%|▋         | 10/150 [00:03<00:42,  3.32it/s]\u001b[A\n",
      "Evaluating workflow:   9%|▊         | 13/150 [00:03<00:25,  5.39it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  10%|█         | 15/150 [00:03<00:19,  6.81it/s]\u001b[A\n",
      "Evaluating workflow:  11%|█▏        | 17/150 [00:03<00:17,  7.80it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  13%|█▎        | 19/150 [00:03<00:15,  8.59it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  14%|█▍        | 21/150 [00:03<00:13,  9.77it/s]\u001b[A\n",
      "Evaluating workflow:  15%|█▌        | 23/150 [00:03<00:11, 10.95it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  17%|█▋        | 26/150 [00:04<00:09, 13.02it/s]\u001b[A\n",
      "Evaluating workflow:  19%|█▊        | 28/150 [00:04<00:09, 13.25it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  20%|██        | 30/150 [00:04<00:17,  6.97it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  21%|██▏       | 32/150 [00:05<00:15,  7.57it/s]\u001b[A\n",
      "Evaluating workflow:  23%|██▎       | 34/150 [00:05<00:13,  8.42it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  24%|██▍       | 36/150 [00:05<00:21,  5.37it/s]\u001b[A\n",
      "Evaluating workflow:  25%|██▍       | 37/150 [00:06<00:20,  5.46it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  25%|██▌       | 38/150 [00:06<00:18,  5.99it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  26%|██▌       | 39/150 [00:06<00:20,  5.41it/s]\u001b[A\n",
      "Evaluating workflow:  27%|██▋       | 41/150 [00:06<00:14,  7.28it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  29%|██▊       | 43/150 [00:06<00:12,  8.45it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  30%|███       | 45/150 [00:06<00:12,  8.72it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  31%|███▏      | 47/150 [00:07<00:12,  8.45it/s]\u001b[A\n",
      "Evaluating workflow:  32%|███▏      | 48/150 [00:07<00:11,  8.54it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  33%|███▎      | 50/150 [00:07<00:09, 10.42it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  36%|███▌      | 54/150 [00:07<00:08, 10.92it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  37%|███▋      | 56/150 [00:08<00:08, 10.57it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  39%|███▊      | 58/150 [00:08<00:11,  8.32it/s]\u001b[A\n",
      "Evaluating workflow:  39%|███▉      | 59/150 [00:08<00:11,  7.77it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  40%|████      | 60/150 [00:08<00:15,  5.92it/s]\u001b[A\n",
      "Evaluating workflow:  41%|████▏     | 62/150 [00:09<00:11,  7.52it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      " {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  43%|████▎     | 64/150 [00:09<00:10,  8.52it/s]\u001b[A\n",
      "Evaluating workflow:  44%|████▍     | 66/150 [00:09<00:08, 10.13it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  45%|████▌     | 68/150 [00:09<00:14,  5.79it/s]\u001b[A\n",
      "Evaluating workflow:  47%|████▋     | 71/150 [00:10<00:09,  8.42it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  49%|████▊     | 73/150 [00:10<00:08,  9.02it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  51%|█████▏    | 77/150 [00:10<00:08,  9.12it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  53%|█████▎    | 79/150 [00:10<00:08,  8.87it/s]\u001b[A\n",
      "Evaluating workflow:  55%|█████▌    | 83/150 [00:11<00:05, 11.76it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  57%|█████▋    | 85/150 [00:11<00:07,  9.09it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  58%|█████▊    | 87/150 [00:11<00:06,  9.70it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  59%|█████▉    | 89/150 [00:11<00:06,  8.89it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  61%|██████    | 91/150 [00:12<00:08,  6.92it/s]\u001b[A\n",
      "Evaluating workflow:  62%|██████▏   | 93/150 [00:12<00:07,  8.13it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  63%|██████▎   | 95/150 [00:12<00:07,  6.94it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  65%|██████▍   | 97/150 [00:13<00:07,  7.18it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  66%|██████▌   | 99/150 [00:13<00:05,  8.79it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  67%|██████▋   | 101/150 [00:13<00:05,  8.57it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  69%|██████▊   | 103/150 [00:13<00:05,  8.05it/s]\u001b[A\n",
      "Evaluating workflow:  70%|███████   | 105/150 [00:14<00:05,  8.68it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  72%|███████▏  | 108/150 [00:14<00:04,  9.55it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  73%|███████▎  | 110/150 [00:14<00:04,  8.07it/s]\u001b[A\n",
      "Evaluating workflow:  74%|███████▍  | 111/150 [00:14<00:05,  7.48it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  75%|███████▍  | 112/150 [00:14<00:04,  7.66it/s]\u001b[A\n",
      "Evaluating workflow:  76%|███████▌  | 114/150 [00:15<00:04,  8.66it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  77%|███████▋  | 115/150 [00:15<00:04,  7.88it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  79%|███████▊  | 118/150 [00:16<00:06,  5.29it/s]\u001b[A\n",
      "Evaluating workflow:  79%|███████▉  | 119/150 [00:16<00:05,  5.76it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  80%|████████  | 120/150 [00:16<00:05,  5.46it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  81%|████████  | 121/150 [00:16<00:06,  4.59it/s]\u001b[A\n",
      "Evaluating workflow:  83%|████████▎ | 124/150 [00:16<00:03,  7.18it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  83%|████████▎ | 125/150 [00:17<00:03,  7.21it/s]\u001b[A\n",
      "Evaluating workflow:  85%|████████▍ | 127/150 [00:17<00:02,  8.64it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  86%|████████▌ | 129/150 [00:17<00:02,  9.13it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  88%|████████▊ | 132/150 [00:17<00:02,  8.55it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  89%|████████▉ | 134/150 [00:18<00:01,  8.24it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  91%|█████████▏| 137/150 [00:18<00:01,  9.54it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  93%|█████████▎| 139/150 [00:18<00:01,  7.43it/s]\u001b[A\n",
      "Evaluating workflow:  93%|█████████▎| 140/150 [00:18<00:01,  6.95it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  95%|█████████▍| 142/150 [00:19<00:01,  7.17it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  96%|█████████▌| 144/150 [00:19<00:01,  4.78it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  97%|█████████▋| 145/150 [00:20<00:01,  4.45it/s]\u001b[A\n",
      "Evaluating workflow:  98%|█████████▊| 147/150 [00:20<00:00,  5.93it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  99%|█████████▊| 148/150 [00:20<00:00,  5.20it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 150/150 [00:21<00:00,  7.08it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2026-01-13 09:29:33.792\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 19 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.5266666666666666}\u001b[0m\n",
      "\u001b[32m2026-01-13 09:29:33.792\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m291\u001b[0m - \u001b[1mMetrics are worse than the best snapshot which has {'f1': 0.0, 'em': 0.0, 'acc': 0.58}. Rolling back to the best snapshot.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 95%|█████████▌| 19/20 [54:36<02:48, 168.43s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 09:29:33.795\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2026-01-13 09:29:50.428\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2026-01-13 09:31:40.305\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2026-01-13 09:31:51.839\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2026-01-13 09:31:51.839\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 20 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/150 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   1%|          | 1/150 [00:01<04:07,  1.66s/it]\u001b[A\n",
      "Evaluating workflow:   2%|▏         | 3/150 [00:01<01:11,  2.06it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   3%|▎         | 5/150 [00:02<01:10,  2.06it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   5%|▍         | 7/150 [00:03<00:48,  2.93it/s]\u001b[A\n",
      "Evaluating workflow:   6%|▌         | 9/150 [00:03<00:32,  4.30it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   7%|▋         | 11/150 [00:03<00:25,  5.52it/s]\u001b[A\n",
      "Evaluating workflow:   9%|▊         | 13/150 [00:03<00:18,  7.25it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  10%|█         | 15/150 [00:03<00:16,  7.97it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  12%|█▏        | 18/150 [00:03<00:14,  8.89it/s]\u001b[A\n",
      "Evaluating workflow:  13%|█▎        | 20/150 [00:04<00:13,  9.84it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  15%|█▌        | 23/150 [00:04<00:11, 11.11it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  17%|█▋        | 25/150 [00:04<00:17,  7.06it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  18%|█▊        | 27/150 [00:05<00:16,  7.52it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  19%|█▊        | 28/150 [00:05<00:19,  6.41it/s]\u001b[A\n",
      "Evaluating workflow:  19%|█▉        | 29/150 [00:05<00:18,  6.59it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  20%|██        | 30/150 [00:05<00:25,  4.71it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  22%|██▏       | 33/150 [00:06<00:20,  5.75it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  23%|██▎       | 35/150 [00:06<00:20,  5.65it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  24%|██▍       | 36/150 [00:06<00:21,  5.26it/s]\u001b[A\n",
      "Evaluating workflow:  25%|██▍       | 37/150 [00:07<00:20,  5.41it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  27%|██▋       | 40/150 [00:07<00:12,  8.62it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  28%|██▊       | 42/150 [00:07<00:11,  9.60it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  29%|██▉       | 44/150 [00:07<00:14,  7.16it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  30%|███       | 45/150 [00:07<00:15,  6.78it/s]\u001b[A\n",
      "Evaluating workflow:  31%|███       | 46/150 [00:08<00:16,  6.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[A\n",
      "Evaluating workflow:  32%|███▏      | 48/150 [00:08<00:14,  7.22it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  33%|███▎      | 49/150 [00:08<00:14,  6.83it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  34%|███▍      | 51/150 [00:08<00:14,  6.73it/s]\u001b[A\n",
      "Evaluating workflow:  36%|███▌      | 54/150 [00:09<00:10,  8.97it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  37%|███▋      | 55/150 [00:09<00:16,  5.64it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  38%|███▊      | 57/150 [00:09<00:14,  6.57it/s]\u001b[A\n",
      "Evaluating workflow:  39%|███▊      | 58/150 [00:09<00:14,  6.35it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  40%|████      | 60/150 [00:10<00:13,  6.65it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  41%|████      | 61/150 [00:10<00:13,  6.66it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  41%|████▏     | 62/150 [00:10<00:15,  5.71it/s]\u001b[A\n",
      "Evaluating workflow:  43%|████▎     | 64/150 [00:10<00:12,  6.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[A\n",
      "Evaluating workflow:  43%|████▎     | 65/150 [00:11<00:14,  6.04it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  44%|████▍     | 66/150 [00:11<00:16,  5.18it/s]\u001b[A\n",
      "Evaluating workflow:  45%|████▍     | 67/150 [00:11<00:15,  5.32it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  47%|████▋     | 70/150 [00:11<00:10,  7.30it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  48%|████▊     | 72/150 [00:12<00:11,  6.89it/s]\u001b[A\n",
      "Evaluating workflow:  50%|█████     | 75/150 [00:12<00:08,  9.04it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  51%|█████     | 76/150 [00:12<00:08,  8.29it/s]\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 78/150 [00:12<00:08,  8.77it/s]\u001b[A\n",
      "Evaluating workflow:  53%|█████▎    | 79/150 [00:12<00:10,  6.98it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  53%|█████▎    | 80/150 [00:13<00:12,  5.60it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  55%|█████▍    | 82/150 [00:13<00:11,  6.05it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  55%|█████▌    | 83/150 [00:13<00:12,  5.55it/s]\u001b[A\n",
      "Evaluating workflow:  57%|█████▋    | 86/150 [00:13<00:07,  8.69it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  59%|█████▊    | 88/150 [00:14<00:13,  4.55it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  59%|█████▉    | 89/150 [00:15<00:13,  4.60it/s]\u001b[A\n",
      "Evaluating workflow:  61%|██████▏   | 92/150 [00:15<00:08,  7.00it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  63%|██████▎   | 94/150 [00:15<00:07,  7.55it/s]\u001b[A\n",
      "Evaluating workflow:  64%|██████▍   | 96/150 [00:15<00:06,  8.46it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  65%|██████▌   | 98/150 [00:16<00:11,  4.41it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  69%|██████▊   | 103/150 [00:16<00:06,  7.07it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  70%|███████   | 105/150 [00:17<00:06,  7.43it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  71%|███████▏  | 107/150 [00:17<00:05,  7.83it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  73%|███████▎  | 109/150 [00:17<00:06,  6.43it/s]\u001b[A\n",
      "Evaluating workflow:  73%|███████▎  | 110/150 [00:17<00:05,  6.74it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  75%|███████▍  | 112/150 [00:17<00:04,  8.20it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  76%|███████▌  | 114/150 [00:18<00:06,  5.85it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  77%|███████▋  | 115/150 [00:18<00:07,  4.98it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  78%|███████▊  | 117/150 [00:19<00:06,  5.47it/s]\u001b[A\n",
      "Evaluating workflow:  79%|███████▊  | 118/150 [00:19<00:05,  5.90it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  79%|███████▉  | 119/150 [00:19<00:05,  6.11it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  80%|████████  | 120/150 [00:19<00:05,  5.50it/s]\u001b[A\n",
      "Evaluating workflow:  81%|████████  | 121/150 [00:19<00:05,  5.77it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  82%|████████▏ | 123/150 [00:19<00:03,  7.37it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  83%|████████▎ | 124/150 [00:20<00:04,  6.18it/s]\u001b[A\n",
      "Evaluating workflow:  83%|████████▎ | 125/150 [00:20<00:03,  6.77it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  84%|████████▍ | 126/150 [00:20<00:06,  3.99it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  85%|████████▌ | 128/150 [00:21<00:04,  5.23it/s]\u001b[A\n",
      "Evaluating workflow:  87%|████████▋ | 131/150 [00:21<00:02,  8.54it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  89%|████████▊ | 133/150 [00:21<00:01,  9.89it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  90%|█████████ | 135/150 [00:21<00:02,  7.01it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  91%|█████████▏| 137/150 [00:22<00:02,  6.17it/s]\u001b[A\n",
      "Evaluating workflow:  93%|█████████▎| 139/150 [00:22<00:01,  7.21it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  94%|█████████▍| 141/150 [00:22<00:01,  6.93it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  95%|█████████▍| 142/150 [00:22<00:01,  6.21it/s]\u001b[A\n",
      "Evaluating workflow:  96%|█████████▌| 144/150 [00:23<00:00,  7.56it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  97%|█████████▋| 146/150 [00:23<00:00,  6.59it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  98%|█████████▊| 147/150 [00:24<00:01,  2.91it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  99%|█████████▊| 148/150 [00:25<00:00,  2.55it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  99%|█████████▉| 149/150 [00:25<00:00,  2.66it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 150/150 [00:26<00:00,  5.76it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 09:32:17.994\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 20 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.47333333333333333}\u001b[0m\n",
      "\u001b[32m2026-01-13 09:32:17.994\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m291\u001b[0m - \u001b[1mMetrics are worse than the best snapshot which has {'f1': 0.0, 'em': 0.0, 'acc': 0.58}. Rolling back to the best snapshot.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|██████████| 20/20 [57:20<00:00, 172.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 09:32:17.998\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m300\u001b[0m - \u001b[1mReached the maximum number of steps 20. Optimization has finished.\u001b[0m\n",
      "\u001b[32m2026-01-13 09:32:17.998\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.workflow.workflow_graph\u001b[0m:\u001b[36msave_module\u001b[0m:\u001b[36m1204\u001b[0m - \u001b[1mSaving SequentialWorkFlowGraph to ./PertQA_textgrad_final.json\u001b[0m\n",
      "\u001b[32m2026-01-13 09:32:18.109\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.workflow.workflow_graph\u001b[0m:\u001b[36msave_module\u001b[0m:\u001b[36m1204\u001b[0m - \u001b[1mSaving SequentialWorkFlowGraph to ./PertQA_textgrad_best.json\u001b[0m\n",
      "\u001b[32m2026-01-13 09:32:18.110\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mrestore_best_graph\u001b[0m:\u001b[36m448\u001b[0m - \u001b[1mRestored the best graph from snapshot with metrics {'f1': 0.0, 'em': 0.0, 'acc': 0.58}\u001b[0m\n",
      "\u001b[32m2026-01-13 09:32:18.110\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m<module>\u001b[0m:\u001b[36m10\u001b[0m - \u001b[1mEvaluating workflow on test set...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 1/3000 [00:00<46:27,  1.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   0%|          | 6/3000 [00:01<09:47,  5.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      " {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   0%|          | 8/3000 [00:01<08:21,  5.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   0%|          | 10/3000 [00:02<07:08,  6.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   0%|          | 13/3000 [00:03<13:22,  3.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 18/3000 [00:03<06:42,  7.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|          | 24/3000 [00:04<04:54, 10.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|          | 29/3000 [00:04<04:20, 11.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 31/3000 [00:04<05:22,  9.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 33/3000 [00:05<05:28,  9.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|          | 36/3000 [00:05<06:15,  7.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|▏         | 39/3000 [00:05<05:23,  9.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|▏         | 41/3000 [00:06<06:02,  8.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|▏         | 42/3000 [00:06<10:00,  4.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 45/3000 [00:06<07:48,  6.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 49/3000 [00:07<07:03,  6.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 56/3000 [00:07<03:28, 14.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 59/3000 [00:07<03:02, 16.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 62/3000 [00:08<03:29, 14.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 64/3000 [00:08<05:14,  9.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 68/3000 [00:08<04:20, 11.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 70/3000 [00:09<07:09,  6.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▎         | 75/3000 [00:10<07:17,  6.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 77/3000 [00:10<07:29,  6.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 80/3000 [00:11<05:30,  8.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 84/3000 [00:11<05:35,  8.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 86/3000 [00:11<06:03,  8.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 91/3000 [00:12<04:04, 11.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 93/3000 [00:12<07:48,  6.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 96/3000 [00:13<06:48,  7.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 100/3000 [00:13<06:10,  7.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 104/3000 [00:14<05:38,  8.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▎         | 106/3000 [00:14<07:58,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▎         | 107/3000 [00:14<08:58,  5.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▎         | 112/3000 [00:15<05:38,  8.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 114/3000 [00:15<04:43, 10.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 117/3000 [00:15<04:15, 11.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 119/3000 [00:15<05:31,  8.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 121/3000 [00:16<05:58,  8.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 124/3000 [00:16<05:08,  9.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 126/3000 [00:17<08:14,  5.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 130/3000 [00:17<05:51,  8.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 132/3000 [00:17<05:47,  8.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 134/3000 [00:17<06:13,  7.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 135/3000 [00:18<06:32,  7.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▍         | 138/3000 [00:18<06:24,  7.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▍         | 144/3000 [00:19<04:35, 10.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 146/3000 [00:19<04:40, 10.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▌         | 152/3000 [00:19<03:43, 12.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 154/3000 [00:20<04:50,  9.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▌         | 158/3000 [00:20<06:35,  7.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▌         | 164/3000 [00:21<04:15, 11.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 166/3000 [00:21<04:23, 10.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 168/3000 [00:21<05:15,  8.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 171/3000 [00:22<06:06,  7.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 174/3000 [00:22<05:12,  9.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 177/3000 [00:22<05:10,  9.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 180/3000 [00:23<04:13, 11.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 182/3000 [00:23<05:30,  8.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 184/3000 [00:23<06:34,  7.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 186/3000 [00:23<05:27,  8.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▋         | 188/3000 [00:24<06:06,  7.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▋         | 191/3000 [00:24<06:15,  7.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▋         | 195/3000 [00:24<04:22, 10.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 197/3000 [00:25<06:49,  6.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 200/3000 [00:25<06:16,  7.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 205/3000 [00:26<04:00, 11.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 208/3000 [00:26<04:38, 10.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 212/3000 [00:27<05:08,  9.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 214/3000 [00:27<06:06,  7.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 216/3000 [00:27<06:52,  6.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 219/3000 [00:28<05:19,  8.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 221/3000 [00:28<05:22,  8.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 224/3000 [00:28<05:58,  7.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 226/3000 [00:29<07:42,  6.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 229/3000 [00:29<06:18,  7.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 231/3000 [00:29<05:18,  8.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 234/3000 [00:29<04:17, 10.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 236/3000 [00:30<07:09,  6.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 241/3000 [00:31<05:55,  7.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 245/3000 [00:31<04:27, 10.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 247/3000 [00:31<04:11, 10.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 249/3000 [00:31<04:18, 10.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 252/3000 [00:32<06:10,  7.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▊         | 256/3000 [00:32<05:15,  8.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▊         | 258/3000 [00:32<05:44,  7.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▊         | 261/3000 [00:33<05:04,  9.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 263/3000 [00:33<06:23,  7.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 267/3000 [00:34<05:50,  7.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 269/3000 [00:34<05:13,  8.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 271/3000 [00:34<04:54,  9.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 278/3000 [00:34<03:11, 14.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 280/3000 [00:35<04:48,  9.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 284/3000 [00:35<05:12,  8.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 286/3000 [00:36<05:31,  8.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 288/3000 [00:36<05:05,  8.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|▉         | 292/3000 [00:37<06:22,  7.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 294/3000 [00:37<05:20,  8.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 296/3000 [00:37<06:15,  7.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 299/3000 [00:37<05:00,  8.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 301/3000 [00:38<05:08,  8.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|█         | 304/3000 [00:38<05:53,  7.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|█         | 307/3000 [00:38<05:14,  8.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 309/3000 [00:38<04:20, 10.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 311/3000 [00:39<05:02,  8.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|█         | 315/3000 [00:39<04:21, 10.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 317/3000 [00:39<04:49,  9.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 319/3000 [00:40<06:45,  6.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 322/3000 [00:40<06:16,  7.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 325/3000 [00:41<06:39,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 328/3000 [00:41<05:02,  8.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 330/3000 [00:41<04:52,  9.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 333/3000 [00:42<07:31,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█▏        | 338/3000 [00:42<05:08,  8.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█▏        | 342/3000 [00:43<05:47,  7.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 347/3000 [00:44<04:45,  9.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 350/3000 [00:44<04:59,  8.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 353/3000 [00:44<05:16,  8.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 355/3000 [00:45<06:07,  7.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 356/3000 [00:45<07:35,  5.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 358/3000 [00:45<07:20,  6.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 360/3000 [00:46<07:50,  5.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 362/3000 [00:46<06:39,  6.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 368/3000 [00:46<03:53, 11.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 370/3000 [00:46<04:30,  9.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 372/3000 [00:47<04:13, 10.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 374/3000 [00:47<05:52,  7.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 377/3000 [00:48<06:44,  6.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 382/3000 [00:48<04:55,  8.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 383/3000 [00:48<04:49,  9.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 386/3000 [00:48<04:42,  9.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 390/3000 [00:49<06:41,  6.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 391/3000 [00:50<07:26,  5.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 396/3000 [00:50<05:03,  8.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 397/3000 [00:50<05:04,  8.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 400/3000 [00:50<04:50,  8.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 402/3000 [00:51<05:11,  8.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▎        | 405/3000 [00:51<06:01,  7.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▎        | 409/3000 [00:52<04:41,  9.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▎        | 412/3000 [00:52<03:21, 12.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 416/3000 [00:52<03:34, 12.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 418/3000 [00:53<07:14,  5.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 422/3000 [00:53<05:55,  7.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 424/3000 [00:53<05:44,  7.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 425/3000 [00:54<06:53,  6.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 428/3000 [00:54<05:55,  7.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 431/3000 [00:55<07:02,  6.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▍        | 437/3000 [00:55<04:41,  9.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 440/3000 [00:55<03:42, 11.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▍        | 444/3000 [00:56<04:16,  9.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 446/3000 [00:56<04:12, 10.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 448/3000 [00:57<07:14,  5.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 450/3000 [00:57<06:49,  6.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 452/3000 [00:57<06:29,  6.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 453/3000 [00:57<06:38,  6.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 457/3000 [00:58<04:02, 10.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 459/3000 [00:58<04:51,  8.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 461/3000 [00:58<04:32,  9.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 462/3000 [00:58<06:04,  6.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 463/3000 [00:59<06:53,  6.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 466/3000 [00:59<06:44,  6.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 471/3000 [01:00<04:15,  9.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 473/3000 [01:00<03:50, 10.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 475/3000 [01:00<04:55,  8.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 477/3000 [01:00<05:13,  8.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 480/3000 [01:01<04:53,  8.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 482/3000 [01:01<06:19,  6.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 484/3000 [01:01<05:11,  8.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 487/3000 [01:02<05:13,  8.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▋        | 489/3000 [01:03<10:03,  4.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▋        | 493/3000 [01:03<06:27,  6.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 498/3000 [01:03<04:15,  9.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 500/3000 [01:03<04:24,  9.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 503/3000 [01:04<04:26,  9.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 506/3000 [01:04<06:13,  6.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 509/3000 [01:05<04:47,  8.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 512/3000 [01:05<04:51,  8.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 514/3000 [01:05<04:11,  9.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 516/3000 [01:06<05:33,  7.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 521/3000 [01:06<05:04,  8.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 526/3000 [01:07<03:59, 10.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 531/3000 [01:07<04:20,  9.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 534/3000 [01:08<04:15,  9.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 536/3000 [01:08<04:25,  9.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 538/3000 [01:08<05:14,  7.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 541/3000 [01:09<05:03,  8.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 545/3000 [01:09<05:17,  7.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 547/3000 [01:09<04:21,  9.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 549/3000 [01:10<04:34,  8.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 551/3000 [01:10<05:04,  8.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 553/3000 [01:10<05:41,  7.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 554/3000 [01:10<06:10,  6.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 555/3000 [01:11<06:54,  5.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▊        | 562/3000 [01:11<03:33, 11.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 564/3000 [01:11<03:42, 10.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 568/3000 [01:12<04:31,  8.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 571/3000 [01:12<04:14,  9.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 574/3000 [01:13<05:51,  6.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 577/3000 [01:13<04:42,  8.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 579/3000 [01:13<04:34,  8.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 580/3000 [01:13<05:14,  7.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 584/3000 [01:14<04:08,  9.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 586/3000 [01:14<05:52,  6.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 588/3000 [01:14<04:57,  8.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|█▉        | 593/3000 [01:15<03:49, 10.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 595/3000 [01:15<03:56, 10.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 597/3000 [01:15<04:09,  9.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|██        | 600/3000 [01:16<05:06,  7.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 601/3000 [01:16<06:06,  6.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|██        | 607/3000 [01:16<03:45, 10.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 609/3000 [01:17<05:00,  7.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|██        | 613/3000 [01:17<04:42,  8.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 615/3000 [01:18<06:03,  6.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 617/3000 [01:18<07:10,  5.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 623/3000 [01:19<03:38, 10.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 625/3000 [01:19<04:13,  9.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 627/3000 [01:19<03:49, 10.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 629/3000 [01:19<05:36,  7.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 631/3000 [01:20<04:45,  8.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 635/3000 [01:20<04:03,  9.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██▏       | 642/3000 [01:20<02:43, 14.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 645/3000 [01:21<05:26,  7.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 647/3000 [01:22<05:38,  6.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 649/3000 [01:22<05:21,  7.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 651/3000 [01:22<04:53,  8.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 655/3000 [01:22<04:10,  9.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 659/3000 [01:23<04:21,  8.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 664/3000 [01:24<04:22,  8.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 666/3000 [01:24<04:19,  8.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 672/3000 [01:24<02:54, 13.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 674/3000 [01:24<03:34, 10.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 680/3000 [01:26<05:39,  6.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 684/3000 [01:26<04:25,  8.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 687/3000 [01:26<03:21, 11.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 689/3000 [01:27<08:00,  4.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 691/3000 [01:28<06:56,  5.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 697/3000 [01:28<04:16,  8.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 700/3000 [01:28<03:43, 10.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 702/3000 [01:28<04:22,  8.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 704/3000 [01:29<04:17,  8.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▎       | 708/3000 [01:29<05:28,  6.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▎       | 711/3000 [01:30<04:43,  8.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 713/3000 [01:30<04:04,  9.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 716/3000 [01:30<05:13,  7.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 717/3000 [01:31<05:48,  6.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 719/3000 [01:31<06:08,  6.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 724/3000 [01:31<04:06,  9.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 726/3000 [01:32<04:14,  8.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 727/3000 [01:32<04:17,  8.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 729/3000 [01:32<04:47,  7.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▍       | 738/3000 [01:33<03:42, 10.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 740/3000 [01:33<03:49,  9.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▍       | 745/3000 [01:34<04:13,  8.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 747/3000 [01:34<04:46,  7.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▌       | 751/3000 [01:35<04:35,  8.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 754/3000 [01:35<03:29, 10.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 756/3000 [01:36<05:42,  6.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▌       | 760/3000 [01:36<04:26,  8.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 762/3000 [01:36<03:42, 10.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 767/3000 [01:36<03:17, 11.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 769/3000 [01:37<03:32, 10.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 771/3000 [01:37<04:17,  8.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 773/3000 [01:37<04:11,  8.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 775/3000 [01:37<04:08,  8.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 778/3000 [01:38<04:42,  7.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 780/3000 [01:38<03:50,  9.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 782/3000 [01:38<05:28,  6.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 783/3000 [01:39<05:10,  7.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▋       | 788/3000 [01:39<04:03,  9.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▋       | 790/3000 [01:39<03:25, 10.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▋       | 792/3000 [01:40<04:29,  8.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▋       | 794/3000 [01:40<05:20,  6.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 799/3000 [01:40<04:41,  7.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 803/3000 [01:41<03:25, 10.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 807/3000 [01:41<04:43,  7.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 809/3000 [01:42<04:40,  7.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 812/3000 [01:42<03:45,  9.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 817/3000 [01:42<03:53,  9.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 819/3000 [01:43<04:02,  8.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 821/3000 [01:43<05:15,  6.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 823/3000 [01:43<05:05,  7.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 829/3000 [01:44<03:05, 11.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 833/3000 [01:44<02:41, 13.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 835/3000 [01:44<02:33, 14.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 837/3000 [01:45<03:58,  9.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 841/3000 [01:46<06:01,  5.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 842/3000 [01:46<06:08,  5.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 844/3000 [01:46<06:11,  5.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 847/3000 [01:46<05:07,  7.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 850/3000 [01:47<04:45,  7.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 851/3000 [01:47<04:39,  7.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 852/3000 [01:47<05:23,  6.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 854/3000 [01:47<04:49,  7.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▊       | 857/3000 [01:48<04:14,  8.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▊       | 858/3000 [01:48<04:46,  7.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▊       | 859/3000 [01:48<05:37,  6.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 863/3000 [01:49<04:19,  8.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 864/3000 [01:49<04:51,  7.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 869/3000 [01:49<03:26, 10.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 874/3000 [01:50<03:43,  9.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 878/3000 [01:50<03:41,  9.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 880/3000 [01:50<03:38,  9.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 882/3000 [01:51<04:42,  7.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 884/3000 [01:51<06:22,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 886/3000 [01:52<04:52,  7.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|██▉       | 889/3000 [01:52<04:09,  8.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 891/3000 [01:52<05:08,  6.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|██▉       | 895/3000 [01:53<04:23,  8.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 897/3000 [01:53<03:52,  9.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 899/3000 [01:53<03:43,  9.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 901/3000 [01:54<04:38,  7.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|███       | 906/3000 [01:54<03:22, 10.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|███       | 910/3000 [01:54<03:38,  9.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 912/3000 [01:55<03:21, 10.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 916/3000 [01:55<03:02, 11.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 918/3000 [01:55<05:05,  6.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 921/3000 [01:56<05:28,  6.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 925/3000 [01:56<04:06,  8.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 927/3000 [01:57<04:10,  8.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 931/3000 [01:57<03:42,  9.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 933/3000 [01:57<04:22,  7.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 936/3000 [01:58<04:17,  8.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███▏      | 939/3000 [01:58<04:20,  7.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███▏      | 941/3000 [01:58<04:24,  7.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███▏      | 942/3000 [01:59<05:02,  6.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███▏      | 944/3000 [01:59<04:36,  7.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 948/3000 [01:59<03:45,  9.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 950/3000 [02:00<04:12,  8.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 952/3000 [02:00<05:12,  6.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 953/3000 [02:00<05:23,  6.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 954/3000 [02:00<06:19,  5.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 959/3000 [02:01<04:04,  8.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 963/3000 [02:01<03:07, 10.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 965/3000 [02:01<03:01, 11.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 967/3000 [02:02<04:20,  7.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 971/3000 [02:03<05:22,  6.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 973/3000 [02:03<04:57,  6.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 978/3000 [02:03<03:09, 10.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 980/3000 [02:03<03:46,  8.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 982/3000 [02:04<05:14,  6.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 986/3000 [02:05<04:50,  6.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 990/3000 [02:05<03:35,  9.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 992/3000 [02:05<03:07, 10.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 994/3000 [02:06<05:21,  6.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 998/3000 [02:06<04:07,  8.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 1000/3000 [02:06<03:31,  9.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 1002/3000 [02:06<03:38,  9.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▎      | 1007/3000 [02:07<03:06, 10.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▎      | 1009/3000 [02:07<05:03,  6.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▎      | 1011/3000 [02:08<04:50,  6.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 1017/3000 [02:08<03:23,  9.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1019/3000 [02:08<03:18,  9.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1022/3000 [02:09<03:19,  9.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1024/3000 [02:09<04:06,  8.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1026/3000 [02:09<05:13,  6.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 1030/3000 [02:10<04:32,  7.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 1035/3000 [02:10<03:20,  9.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 1037/3000 [02:11<03:39,  8.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 1039/3000 [02:11<03:59,  8.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▍      | 1042/3000 [02:11<03:44,  8.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 1044/3000 [02:12<03:54,  8.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 1046/3000 [02:12<04:58,  6.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 1048/3000 [02:12<03:59,  8.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 1050/3000 [02:12<03:55,  8.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 1052/3000 [02:13<03:44,  8.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 1054/3000 [02:13<04:20,  7.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▌      | 1056/3000 [02:13<04:51,  6.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 1059/3000 [02:13<03:12, 10.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 1061/3000 [02:14<04:09,  7.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 1066/3000 [02:14<03:01, 10.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1068/3000 [02:15<03:49,  8.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1070/3000 [02:15<05:01,  6.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 1074/3000 [02:15<04:01,  7.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1076/3000 [02:16<03:38,  8.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1080/3000 [02:16<02:42, 11.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1082/3000 [02:16<02:56, 10.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1084/3000 [02:17<04:13,  7.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1085/3000 [02:17<04:39,  6.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1086/3000 [02:17<05:10,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▋      | 1094/3000 [02:18<03:21,  9.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 1098/3000 [02:18<03:10,  9.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1100/3000 [02:18<03:32,  8.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 1103/3000 [02:19<04:30,  7.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 1107/3000 [02:19<03:10,  9.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 1111/3000 [02:20<03:07, 10.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1113/3000 [02:20<02:40, 11.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1116/3000 [02:21<04:45,  6.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 1121/3000 [02:21<03:25,  9.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1123/3000 [02:21<03:20,  9.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1127/3000 [02:22<03:47,  8.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1129/3000 [02:22<05:27,  5.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1133/3000 [02:23<04:13,  7.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1136/3000 [02:23<03:32,  8.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1141/3000 [02:23<03:23,  9.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1143/3000 [02:24<04:01,  7.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1148/3000 [02:24<03:17,  9.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1150/3000 [02:25<03:25,  9.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1154/3000 [02:25<03:07,  9.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▊      | 1158/3000 [02:26<04:19,  7.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▊      | 1161/3000 [02:26<04:02,  7.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 1164/3000 [02:27<03:48,  8.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 1168/3000 [02:27<03:14,  9.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1170/3000 [02:27<03:16,  9.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1172/3000 [02:27<03:30,  8.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1175/3000 [02:28<03:02, 10.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 1178/3000 [02:28<03:32,  8.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1180/3000 [02:29<05:13,  5.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1181/3000 [02:29<06:08,  4.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|███▉      | 1186/3000 [02:29<03:49,  7.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|███▉      | 1188/3000 [02:30<04:10,  7.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|███▉      | 1190/3000 [02:30<04:12,  7.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 1194/3000 [02:30<02:40, 11.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      " {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|███▉      | 1198/3000 [02:31<03:24,  8.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1200/3000 [02:31<03:36,  8.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|████      | 1203/3000 [02:32<03:56,  7.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1206/3000 [02:32<03:52,  7.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|████      | 1210/3000 [02:33<03:55,  7.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1213/3000 [02:33<03:30,  8.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1214/3000 [02:33<04:00,  7.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1216/3000 [02:33<03:43,  7.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1221/3000 [02:34<03:23,  8.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1223/3000 [02:34<03:27,  8.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1225/3000 [02:34<03:49,  7.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1227/3000 [02:35<03:34,  8.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1230/3000 [02:35<03:41,  7.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1232/3000 [02:35<03:07,  9.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1235/3000 [02:36<03:35,  8.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1237/3000 [02:36<04:35,  6.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████▏     | 1240/3000 [02:36<03:06,  9.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████▏     | 1242/3000 [02:36<03:26,  8.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████▏     | 1244/3000 [02:37<03:17,  8.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1246/3000 [02:37<02:52, 10.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1249/3000 [02:37<03:41,  7.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1251/3000 [02:38<03:42,  7.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1256/3000 [02:38<03:28,  8.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1260/3000 [02:38<02:38, 10.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1264/3000 [02:39<03:10,  9.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1266/3000 [02:39<03:07,  9.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1268/3000 [02:39<03:03,  9.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1270/3000 [02:40<03:44,  7.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1274/3000 [02:40<02:49, 10.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1277/3000 [02:40<02:59,  9.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1279/3000 [02:41<03:12,  8.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1281/3000 [02:41<04:31,  6.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1284/3000 [02:41<03:43,  7.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1287/3000 [02:42<03:57,  7.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1290/3000 [02:42<03:10,  8.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1292/3000 [02:43<03:43,  7.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1294/3000 [02:43<04:34,  6.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1299/3000 [02:43<02:32, 11.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1301/3000 [02:44<03:22,  8.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1303/3000 [02:44<02:57,  9.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▎     | 1308/3000 [02:45<03:36,  7.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▎     | 1310/3000 [02:45<04:02,  6.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▎     | 1312/3000 [02:45<03:40,  7.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1314/3000 [02:45<03:23,  8.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1319/3000 [02:46<02:10, 12.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1321/3000 [02:47<05:06,  5.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1325/3000 [02:47<03:57,  7.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1329/3000 [02:47<03:05,  9.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1334/3000 [02:48<02:34, 10.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▍     | 1338/3000 [02:48<03:43,  7.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1340/3000 [02:49<03:31,  7.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1342/3000 [02:49<03:15,  8.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▍     | 1346/3000 [02:50<03:56,  7.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1348/3000 [02:50<04:33,  6.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      " {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1349/3000 [02:51<06:27,  4.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▌     | 1353/3000 [02:51<04:14,  6.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1357/3000 [02:51<02:54,  9.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1359/3000 [02:51<02:55,  9.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1361/3000 [02:52<03:30,  7.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1363/3000 [02:52<03:45,  7.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1365/3000 [02:52<03:40,  7.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1368/3000 [02:53<03:27,  7.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1372/3000 [02:53<02:41, 10.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1374/3000 [02:54<03:43,  7.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1378/3000 [02:54<03:11,  8.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1379/3000 [02:54<03:23,  7.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1384/3000 [02:55<02:50,  9.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1386/3000 [02:55<02:46,  9.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▋     | 1388/3000 [02:55<02:36, 10.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▋     | 1394/3000 [02:56<03:06,  8.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1396/3000 [02:56<02:51,  9.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1398/3000 [02:56<03:08,  8.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1401/3000 [02:57<02:52,  9.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1403/3000 [02:57<04:07,  6.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1405/3000 [02:57<04:03,  6.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1406/3000 [02:58<04:27,  5.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1409/3000 [02:58<03:53,  6.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1413/3000 [02:58<02:50,  9.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1416/3000 [02:59<02:55,  9.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1420/3000 [02:59<02:59,  8.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1422/3000 [02:59<02:32, 10.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1424/3000 [03:00<02:43,  9.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1426/3000 [03:00<03:14,  8.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1428/3000 [03:00<03:25,  7.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1430/3000 [03:01<03:34,  7.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1432/3000 [03:01<03:28,  7.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1434/3000 [03:01<03:15,  7.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1436/3000 [03:01<04:15,  6.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1438/3000 [03:02<04:27,  5.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1443/3000 [03:02<02:10, 11.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1447/3000 [03:02<01:48, 14.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1451/3000 [03:03<01:52, 13.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1453/3000 [03:03<04:20,  5.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1455/3000 [03:04<04:26,  5.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▊     | 1457/3000 [03:04<03:38,  7.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▊     | 1459/3000 [03:04<03:20,  7.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1463/3000 [03:05<03:38,  7.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1465/3000 [03:05<03:31,  7.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1466/3000 [03:05<03:45,  6.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1468/3000 [03:06<04:09,  6.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1473/3000 [03:06<02:36,  9.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1475/3000 [03:06<02:50,  8.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1476/3000 [03:07<03:39,  6.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1478/3000 [03:07<03:38,  6.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1482/3000 [03:07<03:21,  7.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1483/3000 [03:08<03:20,  7.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1484/3000 [03:08<04:42,  5.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1487/3000 [03:08<03:32,  7.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1489/3000 [03:08<03:25,  7.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1491/3000 [03:09<03:24,  7.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|████▉     | 1497/3000 [03:09<02:03, 12.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1499/3000 [03:10<03:00,  8.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 1502/3000 [03:10<04:03,  6.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 1506/3000 [03:11<03:32,  7.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 1510/3000 [03:11<02:25, 10.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1512/3000 [03:11<02:32,  9.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1514/3000 [03:11<02:22, 10.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1516/3000 [03:12<05:07,  4.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1520/3000 [03:13<03:46,  6.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1524/3000 [03:13<02:30,  9.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1528/3000 [03:13<02:12, 11.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1530/3000 [03:14<02:32,  9.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1532/3000 [03:14<03:28,  7.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1534/3000 [03:14<03:34,  6.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████▏    | 1539/3000 [03:15<02:24, 10.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████▏    | 1542/3000 [03:15<02:38,  9.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████▏    | 1544/3000 [03:15<02:59,  8.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1545/3000 [03:16<03:46,  6.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1550/3000 [03:16<03:24,  7.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1552/3000 [03:17<02:55,  8.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1556/3000 [03:17<02:37,  9.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1560/3000 [03:17<01:57, 12.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1564/3000 [03:18<02:05, 11.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1566/3000 [03:18<03:04,  7.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1571/3000 [03:19<02:49,  8.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1574/3000 [03:19<02:45,  8.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1576/3000 [03:19<03:02,  7.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1578/3000 [03:20<03:18,  7.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1579/3000 [03:20<04:19,  5.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1581/3000 [03:20<04:31,  5.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1587/3000 [03:21<02:12, 10.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1591/3000 [03:21<01:43, 13.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1593/3000 [03:21<02:14, 10.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1595/3000 [03:22<02:23,  9.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1597/3000 [03:22<02:33,  9.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1599/3000 [03:22<03:04,  7.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1601/3000 [03:22<02:49,  8.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1603/3000 [03:23<03:06,  7.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▎    | 1606/3000 [03:23<03:44,  6.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▎    | 1608/3000 [03:24<03:32,  6.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▎    | 1609/3000 [03:24<03:35,  6.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▎    | 1611/3000 [03:24<03:13,  7.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▎    | 1612/3000 [03:24<03:52,  5.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1615/3000 [03:25<02:52,  8.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1619/3000 [03:25<02:33,  8.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1621/3000 [03:25<02:49,  8.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1623/3000 [03:26<03:48,  6.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1625/3000 [03:26<03:53,  5.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1629/3000 [03:26<02:23,  9.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1631/3000 [03:27<02:45,  8.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1634/3000 [03:27<02:48,  8.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▍    | 1637/3000 [03:28<03:59,  5.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▍    | 1639/3000 [03:28<04:05,  5.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▍    | 1641/3000 [03:29<03:49,  5.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▍    | 1646/3000 [03:29<02:31,  8.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1648/3000 [03:29<02:18,  9.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▌    | 1651/3000 [03:30<02:45,  8.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1653/3000 [03:30<02:50,  7.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▌    | 1655/3000 [03:31<04:20,  5.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1658/3000 [03:31<02:43,  8.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1662/3000 [03:31<02:09, 10.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1665/3000 [03:32<04:04,  5.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1668/3000 [03:32<03:09,  7.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1671/3000 [03:33<02:32,  8.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1673/3000 [03:33<02:37,  8.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1677/3000 [03:34<02:53,  7.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1679/3000 [03:34<02:29,  8.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1682/3000 [03:34<01:57, 11.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1684/3000 [03:34<02:15,  9.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▋    | 1688/3000 [03:35<02:15,  9.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▋    | 1690/3000 [03:35<02:05, 10.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▋    | 1694/3000 [03:36<03:07,  6.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1696/3000 [03:36<03:09,  6.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1701/3000 [03:36<02:30,  8.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1703/3000 [03:37<03:18,  6.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1704/3000 [03:37<03:40,  5.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1708/3000 [03:37<02:40,  8.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1710/3000 [03:38<02:12,  9.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1714/3000 [03:38<02:10,  9.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1716/3000 [03:38<02:31,  8.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1717/3000 [03:39<03:04,  6.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      " {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1720/3000 [03:39<02:55,  7.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1724/3000 [03:39<01:53, 11.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1726/3000 [03:40<03:03,  6.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1729/3000 [03:40<03:13,  6.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1732/3000 [03:40<02:32,  8.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1737/3000 [03:41<02:24,  8.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1739/3000 [03:41<02:23,  8.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1741/3000 [03:42<02:37,  8.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1743/3000 [03:42<03:00,  6.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1745/3000 [03:42<03:10,  6.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1749/3000 [03:43<02:46,  7.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1752/3000 [03:43<02:45,  7.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1753/3000 [03:43<02:38,  7.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1754/3000 [03:44<03:42,  5.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▊    | 1756/3000 [03:44<03:25,  6.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▊    | 1758/3000 [03:44<03:17,  6.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1763/3000 [03:45<02:25,  8.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1765/3000 [03:45<03:12,  6.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1766/3000 [03:45<03:16,  6.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1769/3000 [03:46<02:34,  7.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1772/3000 [03:46<02:16,  9.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1777/3000 [03:46<02:05,  9.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1779/3000 [03:47<02:21,  8.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1781/3000 [03:47<02:58,  6.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1784/3000 [03:48<02:54,  6.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1785/3000 [03:48<03:44,  5.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|█████▉    | 1788/3000 [03:49<03:25,  5.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|█████▉    | 1794/3000 [03:49<02:26,  8.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1796/3000 [03:49<02:07,  9.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|█████▉    | 1799/3000 [03:50<03:16,  6.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1801/3000 [03:50<02:40,  7.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|██████    | 1805/3000 [03:51<02:15,  8.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1807/3000 [03:51<02:05,  9.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|██████    | 1811/3000 [03:51<01:49, 10.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1813/3000 [03:51<01:35, 12.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1815/3000 [03:52<02:39,  7.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1821/3000 [03:52<02:12,  8.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1824/3000 [03:53<01:49, 10.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1826/3000 [03:53<03:00,  6.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1828/3000 [03:54<02:49,  6.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1830/3000 [03:54<03:08,  6.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1831/3000 [03:54<03:04,  6.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1837/3000 [03:54<01:55, 10.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████▏   | 1841/3000 [03:55<01:29, 12.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████▏   | 1843/3000 [03:55<01:54, 10.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1847/3000 [03:56<02:11,  8.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1850/3000 [03:56<02:51,  6.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1852/3000 [03:57<03:00,  6.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1854/3000 [03:57<02:51,  6.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1857/3000 [03:57<02:43,  6.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1863/3000 [03:58<01:40, 11.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1865/3000 [03:58<01:52, 10.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1867/3000 [03:58<01:51, 10.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1869/3000 [03:58<02:02,  9.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1871/3000 [03:59<02:26,  7.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1873/3000 [03:59<02:20,  8.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1876/3000 [03:59<02:35,  7.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1877/3000 [04:00<02:52,  6.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1881/3000 [04:00<02:05,  8.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1883/3000 [04:00<02:42,  6.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1886/3000 [04:01<02:31,  7.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1888/3000 [04:01<02:34,  7.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1891/3000 [04:01<02:00,  9.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1896/3000 [04:02<01:50, 10.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1900/3000 [04:02<01:40, 10.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1902/3000 [04:03<02:36,  7.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1904/3000 [04:03<03:03,  5.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▎   | 1906/3000 [04:03<02:53,  6.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▎   | 1908/3000 [04:04<04:27,  4.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▎   | 1911/3000 [04:05<03:02,  5.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▎   | 1912/3000 [04:05<03:06,  5.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1915/3000 [04:06<03:08,  5.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1919/3000 [04:06<02:07,  8.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1921/3000 [04:06<02:11,  8.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1924/3000 [04:07<02:30,  7.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1926/3000 [04:07<02:07,  8.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1929/3000 [04:07<02:37,  6.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1932/3000 [04:07<01:47,  9.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1935/3000 [04:08<02:06,  8.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1937/3000 [04:08<02:38,  6.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1938/3000 [04:09<02:49,  6.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▍   | 1940/3000 [04:09<03:36,  4.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1942/3000 [04:09<03:04,  5.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▍   | 1947/3000 [04:10<01:49,  9.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 1952/3000 [04:11<02:45,  6.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1954/3000 [04:11<03:05,  5.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1957/3000 [04:12<02:30,  6.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1959/3000 [04:12<02:11,  7.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 1964/3000 [04:12<01:26, 11.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1966/3000 [04:13<02:46,  6.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1968/3000 [04:13<03:25,  5.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1970/3000 [04:14<03:12,  5.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1971/3000 [04:14<02:54,  5.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1976/3000 [04:14<02:01,  8.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1979/3000 [04:14<01:26, 11.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1983/3000 [04:15<01:32, 11.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1985/3000 [04:15<02:38,  6.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1987/3000 [04:16<03:23,  4.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▋   | 1989/3000 [04:16<02:53,  5.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▋   | 1994/3000 [04:17<02:02,  8.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 1996/3000 [04:17<02:04,  8.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 1999/3000 [04:18<02:33,  6.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2001/3000 [04:18<02:04,  8.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2003/3000 [04:18<02:03,  8.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2005/3000 [04:18<02:24,  6.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 2010/3000 [04:19<01:45,  9.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2012/3000 [04:19<01:30, 10.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2014/3000 [04:19<02:02,  8.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2016/3000 [04:20<02:49,  5.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2019/3000 [04:20<02:09,  7.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2021/3000 [04:20<02:05,  7.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2022/3000 [04:20<02:24,  6.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2024/3000 [04:21<02:22,  6.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2027/3000 [04:21<02:43,  5.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2030/3000 [04:22<02:03,  7.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2033/3000 [04:22<01:58,  8.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2037/3000 [04:22<01:33, 10.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2039/3000 [04:23<01:59,  8.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2041/3000 [04:23<01:52,  8.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2043/3000 [04:23<02:00,  7.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2047/3000 [04:24<02:14,  7.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2051/3000 [04:24<01:35,  9.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2053/3000 [04:25<02:18,  6.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2055/3000 [04:25<02:10,  7.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▊   | 2059/3000 [04:25<02:04,  7.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▊   | 2061/3000 [04:26<01:49,  8.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 2064/3000 [04:26<02:04,  7.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 2068/3000 [04:27<02:00,  7.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 2070/3000 [04:27<02:25,  6.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2075/3000 [04:27<01:21, 11.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2077/3000 [04:28<01:46,  8.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2079/3000 [04:28<01:33,  9.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 2083/3000 [04:28<01:48,  8.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|██████▉   | 2086/3000 [04:29<01:54,  8.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|██████▉   | 2088/3000 [04:29<02:06,  7.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|██████▉   | 2092/3000 [04:30<02:02,  7.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 2094/3000 [04:30<03:04,  4.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|██████▉   | 2098/3000 [04:31<02:20,  6.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 2102/3000 [04:31<01:24, 10.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 2104/3000 [04:32<02:11,  6.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|███████   | 2109/3000 [04:32<01:27, 10.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|███████   | 2113/3000 [04:32<01:43,  8.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 2115/3000 [04:33<01:33,  9.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2118/3000 [04:33<01:15, 11.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2120/3000 [04:33<01:59,  7.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2123/3000 [04:34<01:43,  8.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2125/3000 [04:34<02:27,  5.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 2128/3000 [04:35<02:01,  7.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2130/3000 [04:35<01:47,  8.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 2133/3000 [04:35<02:13,  6.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 2136/3000 [04:36<01:54,  7.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████▏  | 2139/3000 [04:36<01:50,  7.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████▏  | 2141/3000 [04:36<01:57,  7.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████▏  | 2143/3000 [04:37<02:15,  6.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2147/3000 [04:37<01:33,  9.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2150/3000 [04:37<01:25,  9.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2152/3000 [04:38<01:24, 10.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2154/3000 [04:38<02:01,  6.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2158/3000 [04:38<01:34,  8.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2161/3000 [04:39<01:53,  7.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2163/3000 [04:39<01:36,  8.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2165/3000 [04:39<01:38,  8.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2166/3000 [04:39<01:39,  8.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2169/3000 [04:40<02:06,  6.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▎  | 2175/3000 [04:40<01:11, 11.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2177/3000 [04:41<01:18, 10.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2179/3000 [04:41<01:24,  9.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2181/3000 [04:41<01:40,  8.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 2185/3000 [04:42<01:31,  8.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2186/3000 [04:42<01:38,  8.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 2190/3000 [04:42<01:48,  7.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2192/3000 [04:43<01:45,  7.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 2195/3000 [04:43<02:05,  6.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 2197/3000 [04:44<02:07,  6.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 2201/3000 [04:44<01:35,  8.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2203/3000 [04:44<01:20,  9.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▎  | 2205/3000 [04:44<01:37,  8.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▎  | 2209/3000 [04:45<01:28,  8.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▎  | 2212/3000 [04:46<02:22,  5.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 2215/3000 [04:47<02:52,  4.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 2219/3000 [04:47<01:41,  7.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2221/3000 [04:47<01:28,  8.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2224/3000 [04:47<01:30,  8.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2226/3000 [04:48<02:01,  6.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2230/3000 [04:48<01:25,  9.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2232/3000 [04:49<02:11,  5.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▍  | 2236/3000 [04:49<01:45,  7.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2237/3000 [04:50<02:20,  5.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▍  | 2240/3000 [04:50<01:56,  6.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2242/3000 [04:50<01:30,  8.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2244/3000 [04:50<01:27,  8.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2246/3000 [04:51<01:30,  8.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▌  | 2251/3000 [04:51<01:06, 11.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 2253/3000 [04:51<01:14,  9.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 2255/3000 [04:52<01:57,  6.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 2258/3000 [04:52<01:25,  8.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▌  | 2262/3000 [04:52<01:10, 10.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 2264/3000 [04:53<01:24,  8.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2266/3000 [04:53<01:20,  9.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2268/3000 [04:53<01:48,  6.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 2271/3000 [04:54<01:50,  6.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 2273/3000 [04:54<01:38,  7.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 2275/3000 [04:54<01:49,  6.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 2278/3000 [04:55<01:22,  8.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 2282/3000 [04:55<01:08, 10.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2285/3000 [04:55<00:55, 12.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2287/3000 [04:56<01:35,  7.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▋  | 2290/3000 [04:56<01:24,  8.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▋  | 2294/3000 [04:56<01:14,  9.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2296/3000 [04:57<01:18,  8.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 2300/3000 [04:57<01:25,  8.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2303/3000 [04:58<01:39,  7.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 2306/3000 [04:58<01:34,  7.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2307/3000 [04:58<01:29,  7.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 2311/3000 [04:59<01:39,  6.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2312/3000 [04:59<01:57,  5.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2315/3000 [05:00<01:27,  7.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2316/3000 [05:00<01:42,  6.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2317/3000 [05:00<01:58,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 2321/3000 [05:00<01:29,  7.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2326/3000 [05:01<01:07, 10.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2329/3000 [05:01<00:56, 11.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2332/3000 [05:02<01:42,  6.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2333/3000 [05:02<01:56,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2337/3000 [05:02<01:15,  8.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2339/3000 [05:03<01:45,  6.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2340/3000 [05:03<01:52,  5.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2342/3000 [05:03<01:48,  6.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2347/3000 [05:04<01:19,  8.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2350/3000 [05:04<01:00, 10.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2352/3000 [05:04<01:09,  9.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2354/3000 [05:05<01:23,  7.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▊  | 2357/3000 [05:06<01:49,  5.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▊  | 2358/3000 [05:06<01:54,  5.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 2364/3000 [05:06<00:58, 10.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2366/3000 [05:07<01:24,  7.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2368/3000 [05:07<01:21,  7.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2370/3000 [05:07<01:32,  6.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 2373/3000 [05:08<01:17,  8.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2376/3000 [05:08<00:56, 11.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2378/3000 [05:08<01:01, 10.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 2383/3000 [05:08<00:52, 11.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 2385/3000 [05:09<01:46,  5.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 2389/3000 [05:09<01:14,  8.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|███████▉  | 2394/3000 [05:10<01:08,  8.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 2397/3000 [05:10<00:55, 10.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 2401/3000 [05:11<01:24,  7.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2403/3000 [05:12<01:37,  6.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2405/3000 [05:12<01:34,  6.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2408/3000 [05:12<01:14,  7.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2410/3000 [05:12<01:16,  7.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2413/3000 [05:12<01:03,  9.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2416/3000 [05:13<01:17,  7.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2418/3000 [05:13<01:05,  8.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2420/3000 [05:13<01:03,  9.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2422/3000 [05:14<01:19,  7.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2425/3000 [05:14<01:23,  6.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2427/3000 [05:14<01:05,  8.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2430/3000 [05:15<01:33,  6.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2431/3000 [05:15<01:27,  6.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2436/3000 [05:16<01:14,  7.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████▏ | 2439/3000 [05:16<01:12,  7.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████▏ | 2443/3000 [05:17<01:06,  8.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████▏ | 2444/3000 [05:17<01:12,  7.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2445/3000 [05:17<01:20,  6.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2448/3000 [05:17<01:13,  7.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2450/3000 [05:17<00:56,  9.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2454/3000 [05:18<00:54,  9.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2458/3000 [05:18<01:06,  8.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2460/3000 [05:19<01:21,  6.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2461/3000 [05:19<01:48,  4.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2468/3000 [05:20<01:08,  7.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2472/3000 [05:21<01:05,  8.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▎ | 2475/3000 [05:21<01:17,  6.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2480/3000 [05:22<01:05,  7.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2484/3000 [05:22<01:00,  8.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2486/3000 [05:22<00:53,  9.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2490/3000 [05:23<00:58,  8.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2492/3000 [05:23<01:04,  7.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2495/3000 [05:24<00:53,  9.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2497/3000 [05:24<01:07,  7.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2501/3000 [05:24<01:01,  8.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2503/3000 [05:25<00:58,  8.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▎ | 2505/3000 [05:25<01:05,  7.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▎ | 2508/3000 [05:25<00:50,  9.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▎ | 2510/3000 [05:25<00:56,  8.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▎ | 2512/3000 [05:26<01:22,  5.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2517/3000 [05:26<00:55,  8.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2519/3000 [05:27<00:58,  8.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2523/3000 [05:27<00:39, 12.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2525/3000 [05:27<00:43, 10.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2527/3000 [05:27<00:45, 10.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2529/3000 [05:28<01:12,  6.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2532/3000 [05:28<01:06,  7.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2534/3000 [05:28<00:53,  8.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▍ | 2539/3000 [05:29<00:44, 10.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▍ | 2543/3000 [05:30<01:08,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▍ | 2549/3000 [05:30<00:38, 11.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2552/3000 [05:30<00:32, 13.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2554/3000 [05:30<00:36, 12.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2556/3000 [05:31<00:40, 11.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2558/3000 [05:31<00:53,  8.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2560/3000 [05:31<00:56,  7.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2562/3000 [05:32<00:52,  8.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2567/3000 [05:32<00:43,  9.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2570/3000 [05:32<00:37, 11.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2572/3000 [05:32<00:44,  9.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2577/3000 [05:34<01:00,  7.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2580/3000 [05:34<00:49,  8.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2582/3000 [05:34<00:48,  8.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2584/3000 [05:34<00:57,  7.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2585/3000 [05:35<00:58,  7.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▋ | 2589/3000 [05:35<00:49,  8.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▋ | 2591/3000 [05:35<00:50,  8.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▋ | 2594/3000 [05:35<00:39, 10.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2596/3000 [05:36<00:53,  7.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2598/3000 [05:36<00:46,  8.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2601/3000 [05:36<00:50,  7.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2603/3000 [05:37<00:44,  8.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2607/3000 [05:37<00:46,  8.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2609/3000 [05:37<00:41,  9.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2613/3000 [05:38<00:42,  9.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2615/3000 [05:38<00:39,  9.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2617/3000 [05:38<00:43,  8.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2620/3000 [05:39<00:51,  7.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2624/3000 [05:39<00:43,  8.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2626/3000 [05:40<00:47,  7.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2628/3000 [05:40<00:51,  7.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2629/3000 [05:40<00:52,  7.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2631/3000 [05:40<00:56,  6.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2637/3000 [05:41<00:32, 11.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2639/3000 [05:41<00:48,  7.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2642/3000 [05:42<00:42,  8.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2644/3000 [05:42<00:44,  8.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2645/3000 [05:42<00:52,  6.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2648/3000 [05:43<00:48,  7.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2650/3000 [05:43<00:51,  6.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2651/3000 [05:43<01:02,  5.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▊ | 2656/3000 [05:44<00:37,  9.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▊ | 2658/3000 [05:44<00:31, 10.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▊ | 2662/3000 [05:44<00:28, 11.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2664/3000 [05:44<00:32, 10.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2666/3000 [05:45<00:46,  7.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2668/3000 [05:45<00:56,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2671/3000 [05:45<00:36,  8.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2673/3000 [05:46<00:34,  9.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2675/3000 [05:46<00:37,  8.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2677/3000 [05:46<00:49,  6.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2681/3000 [05:47<00:46,  6.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2683/3000 [05:47<00:42,  7.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 2686/3000 [05:47<00:38,  8.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|████████▉ | 2690/3000 [05:48<00:33,  9.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|████████▉ | 2698/3000 [05:49<00:46,  6.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2702/3000 [05:49<00:30,  9.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2705/3000 [05:50<00:36,  8.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2707/3000 [05:50<00:31,  9.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2709/3000 [05:50<00:33,  8.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2711/3000 [05:51<00:40,  7.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|█████████ | 2714/3000 [05:51<00:41,  6.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2717/3000 [05:52<00:41,  6.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2721/3000 [05:52<00:27, 10.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2726/3000 [05:53<00:39,  6.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2728/3000 [05:53<00:47,  5.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2730/3000 [05:54<00:39,  6.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2733/3000 [05:54<00:29,  8.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2736/3000 [05:54<00:31,  8.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████▏| 2739/3000 [05:55<00:41,  6.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████▏| 2740/3000 [05:55<00:40,  6.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████▏| 2742/3000 [05:55<00:35,  7.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2749/3000 [05:56<00:23, 10.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2751/3000 [05:56<00:33,  7.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2753/3000 [05:56<00:29,  8.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2755/3000 [05:57<00:40,  6.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2757/3000 [05:57<00:36,  6.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2760/3000 [05:58<00:31,  7.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2763/3000 [05:58<00:31,  7.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2768/3000 [05:58<00:22, 10.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2770/3000 [05:59<00:26,  8.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2772/3000 [05:59<00:34,  6.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2776/3000 [06:00<00:28,  7.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2779/3000 [06:00<00:20, 10.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2781/3000 [06:00<00:23,  9.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2784/3000 [06:00<00:20, 10.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2786/3000 [06:01<00:31,  6.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2788/3000 [06:01<00:33,  6.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2790/3000 [06:01<00:30,  6.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2794/3000 [06:02<00:26,  7.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2796/3000 [06:02<00:22,  9.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2799/3000 [06:02<00:20,  9.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2801/3000 [06:03<00:22,  8.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2804/3000 [06:03<00:23,  8.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▎| 2805/3000 [06:03<00:26,  7.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▎| 2810/3000 [06:04<00:21,  8.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▎| 2812/3000 [06:04<00:22,  8.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2815/3000 [06:04<00:17, 10.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2817/3000 [06:05<00:24,  7.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2822/3000 [06:05<00:19,  9.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2826/3000 [06:06<00:23,  7.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2828/3000 [06:06<00:22,  7.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2830/3000 [06:06<00:18,  9.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2832/3000 [06:07<00:18,  8.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2834/3000 [06:07<00:21,  7.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 2837/3000 [06:07<00:17,  9.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▍| 2841/3000 [06:07<00:15, 10.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▍| 2845/3000 [06:08<00:14, 10.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 2847/3000 [06:09<00:26,  5.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▌| 2851/3000 [06:09<00:20,  7.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▌| 2855/3000 [06:10<00:17,  8.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▌| 2861/3000 [06:10<00:13, 10.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2863/3000 [06:10<00:17,  7.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2865/3000 [06:11<00:15,  8.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2869/3000 [06:11<00:14,  9.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2872/3000 [06:11<00:12, 10.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2874/3000 [06:12<00:15,  8.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2875/3000 [06:12<00:18,  6.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2876/3000 [06:12<00:19,  6.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2880/3000 [06:13<00:18,  6.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2881/3000 [06:13<00:17,  6.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2885/3000 [06:13<00:13,  8.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▋| 2889/3000 [06:14<00:10, 10.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▋| 2891/3000 [06:14<00:10,  9.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2896/3000 [06:14<00:10,  9.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2900/3000 [06:15<00:12,  8.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2904/3000 [06:15<00:09, 10.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2906/3000 [06:16<00:14,  6.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2911/3000 [06:17<00:12,  6.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2915/3000 [06:17<00:09,  8.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2918/3000 [06:17<00:07, 11.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2920/3000 [06:18<00:09,  8.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2925/3000 [06:18<00:06, 11.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2927/3000 [06:18<00:08,  8.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2929/3000 [06:19<00:10,  6.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2931/3000 [06:19<00:08,  7.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2933/3000 [06:19<00:10,  6.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2934/3000 [06:20<00:12,  5.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2938/3000 [06:20<00:08,  7.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2940/3000 [06:20<00:07,  7.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2944/3000 [06:21<00:05,  9.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2947/3000 [06:21<00:06,  7.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2949/3000 [06:22<00:06,  7.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2952/3000 [06:22<00:06,  7.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2954/3000 [06:22<00:04,  9.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▊| 2959/3000 [06:22<00:03, 11.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▊| 2961/3000 [06:23<00:03, 12.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2963/3000 [06:23<00:04,  8.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2966/3000 [06:24<00:04,  7.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2967/3000 [06:24<00:04,  7.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2973/3000 [06:24<00:02, 10.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2977/3000 [06:25<00:03,  7.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2980/3000 [06:25<00:02,  6.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2984/3000 [06:26<00:01, 11.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2986/3000 [06:26<00:01, 10.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2990/3000 [06:27<00:01,  7.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2992/3000 [06:27<00:01,  7.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2993/3000 [06:27<00:01,  6.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2995/3000 [06:27<00:00,  7.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2997/3000 [06:28<00:00,  5.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2998/3000 [06:28<00:00,  5.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|██████████| 3000/3000 [06:29<00:00,  7.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 09:38:47.314\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m<module>\u001b[0m:\u001b[36m13\u001b[0m - \u001b[1mEvaluation metrics (after optimization): {'f1': 0.0, 'em': 0.0, 'acc': 0.49833333333333335}\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "logger.info(\"Evaluating workflow on test set...\")\n",
    "with suppress_logger_info():\n",
    "    results = textgrad_optimizer.evaluate(dataset=benchmark, eval_mode=\"test\")\n",
    "logger.info(f\"Evaluation metrics (before optimization): {results}\")\n",
    "\n",
    "logger.info(\"Optimizing workflow...\")\n",
    "textgrad_optimizer.optimize(benchmark, seed=8)\n",
    "textgrad_optimizer.restore_best_graph()\n",
    "\n",
    "logger.info(\"Evaluating workflow on test set...\")\n",
    "with suppress_logger_info():\n",
    "    results = textgrad_optimizer.evaluate(dataset=benchmark, eval_mode=\"test\")\n",
    "logger.info(f\"Evaluation metrics (after optimization): {results}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "4b6f274d",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'f1': 0.0, 'em': 0.0, 'acc': 0.49833333333333335}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "8ffcb068",
   "metadata": {},
   "outputs": [],
   "source": [
    "#4o generate workflow\n",
    "qa_workflow = {\n",
    "    \"goal\": \"Provide a direct answer to the question based on the context, without including explanations or reasoning.\",\n",
    "    \"tasks\": [\n",
    "        {\n",
    "            \"name\": \"answer_generate\",\n",
    "            \"description\": \"Generate a direct answer to the question based on the context.\",\n",
    "            \"inputs\": [\n",
    "                {\"name\": \"problem\", \"type\": \"str\", \"required\": True, \"description\": \"The question to answer directly.\"}\n",
    "            ],\n",
    "            \"outputs\": [\n",
    "                {\"name\": \"answer\", \"type\": \"str\", \"required\": True, \"description\": \"The direct answer to the question.\"}\n",
    "            ],\n",
    "            \"prompt_template\": StringTemplate(instruction=\"Provide a concise and direct answer to the question.\"),\n",
    "            \"parse_mode\": \"xml\"\n",
    "        }\n",
    "    ] \n",
    "}\n",
    "def collate_func(example: dict) -> dict:\n",
    "    problem = \"Question: {}\\n\\nAnswer:\".format(example[\"question\"])\n",
    "    return {\"problem\": problem}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "dfaf5515",
   "metadata": {},
   "outputs": [],
   "source": [
    "workflow_graph = SequentialWorkFlowGraph.from_dict(qa_workflow)\n",
    "agent_manager = AgentManager()\n",
    "agent_manager.add_agents_from_workflow(workflow_graph, executor_llm.config)\n",
    "\n",
    "evaluator = Evaluator(\n",
    "    llm=executor_llm, \n",
    "    agent_manager=agent_manager, \n",
    "    collate_func=collate_func, \n",
    "    num_workers=20, \n",
    "    verbose=True\n",
    ")\n",
    "\n",
    "textgrad_optimizer = TextGradOptimizer(\n",
    "    graph=workflow_graph, \n",
    "    optimize_mode=\"all\",\n",
    "    executor_llm=executor_llm, \n",
    "    optimizer_llm=optimizer_llm,\n",
    "    batch_size=3,\n",
    "    max_steps=20,\n",
    "    evaluator=evaluator,\n",
    "    eval_every_n_steps=1,\n",
    "    eval_rounds=1,\n",
    "    save_interval=None,\n",
    "    save_path=\"./\",\n",
    "    rollback=True,\n",
    "    constraints=[]\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "2d25086a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3000"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(benchmark._test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "09118f35",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 09:49:33.853\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m<module>\u001b[0m:\u001b[36m1\u001b[0m - \u001b[1mEvaluating workflow on test set...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   0%|          | 3/3000 [00:00<12:28,  4.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   0%|          | 9/3000 [00:01<04:00, 12.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|          | 18/3000 [00:01<02:43, 18.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|          | 23/3000 [00:01<03:19, 14.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 28/3000 [00:02<02:39, 18.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 31/3000 [00:02<02:49, 17.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|▏         | 41/3000 [00:02<02:09, 22.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|▏         | 44/3000 [00:02<02:54, 16.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 51/3000 [00:03<02:15, 21.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 54/3000 [00:03<02:42, 18.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 57/3000 [00:03<02:41, 18.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 66/3000 [00:04<02:18, 21.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 72/3000 [00:04<02:22, 20.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▎         | 75/3000 [00:04<02:50, 17.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 82/3000 [00:04<02:11, 22.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 85/3000 [00:04<02:15, 21.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 88/3000 [00:05<02:33, 18.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 97/3000 [00:05<02:41, 17.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 100/3000 [00:05<02:50, 16.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▎         | 105/3000 [00:06<02:21, 20.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▎         | 112/3000 [00:06<01:58, 24.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 120/3000 [00:06<02:02, 23.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 123/3000 [00:07<02:43, 17.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 129/3000 [00:07<02:04, 23.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 132/3000 [00:07<02:19, 20.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▍         | 138/3000 [00:07<02:15, 21.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 141/3000 [00:07<02:37, 18.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 147/3000 [00:08<02:06, 22.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 150/3000 [00:08<02:15, 21.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▌         | 157/3000 [00:08<02:25, 19.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 161/3000 [00:08<02:19, 20.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 166/3000 [00:09<02:29, 19.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 170/3000 [00:09<02:02, 23.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 176/3000 [00:09<02:26, 19.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 183/3000 [00:09<01:58, 23.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▋         | 189/3000 [00:10<02:33, 18.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 196/3000 [00:10<02:04, 22.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 200/3000 [00:10<01:56, 24.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 203/3000 [00:11<02:46, 16.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 208/3000 [00:11<02:22, 19.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 214/3000 [00:11<02:10, 21.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 218/3000 [00:11<01:53, 24.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 221/3000 [00:11<02:18, 20.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 227/3000 [00:12<02:36, 17.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 229/3000 [00:12<02:49, 16.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 234/3000 [00:12<02:15, 20.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 240/3000 [00:12<02:03, 22.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 246/3000 [00:13<02:19, 19.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 252/3000 [00:13<02:10, 20.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▊         | 257/3000 [00:13<02:07, 21.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▊         | 260/3000 [00:13<02:12, 20.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 263/3000 [00:13<02:23, 19.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 266/3000 [00:14<02:23, 19.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 276/3000 [00:14<01:51, 24.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 279/3000 [00:14<02:16, 19.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 282/3000 [00:14<02:47, 16.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 286/3000 [00:15<02:30, 18.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|▉         | 295/3000 [00:15<01:45, 25.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|█         | 303/3000 [00:15<02:08, 21.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 306/3000 [00:16<02:29, 18.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|█         | 313/3000 [00:16<02:09, 20.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 316/3000 [00:16<02:04, 21.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 323/3000 [00:16<02:03, 21.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 327/3000 [00:17<02:17, 19.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 330/3000 [00:17<02:11, 20.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 336/3000 [00:17<02:18, 19.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█▏        | 339/3000 [00:17<02:29, 17.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█▏        | 342/3000 [00:17<02:21, 18.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 350/3000 [00:18<02:00, 21.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 358/3000 [00:18<02:11, 20.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 365/3000 [00:18<01:58, 22.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 369/3000 [00:19<01:51, 23.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 372/3000 [00:19<02:19, 18.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 376/3000 [00:19<03:21, 13.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 382/3000 [00:20<03:46, 11.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 396/3000 [00:20<02:23, 18.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 399/3000 [00:21<02:14, 19.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 404/3000 [00:21<02:46, 15.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▎        | 411/3000 [00:21<02:08, 20.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 414/3000 [00:21<02:05, 20.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 421/3000 [00:22<01:57, 22.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 427/3000 [00:22<02:31, 16.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 433/3000 [00:23<02:18, 18.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 438/3000 [00:23<01:55, 22.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 441/3000 [00:23<01:58, 21.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▍        | 447/3000 [00:23<02:02, 20.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 450/3000 [00:23<02:07, 19.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 455/3000 [00:24<02:32, 16.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 462/3000 [00:24<01:55, 21.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 465/3000 [00:24<02:04, 20.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 471/3000 [00:24<02:09, 19.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 474/3000 [00:25<02:16, 18.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 479/3000 [00:25<02:32, 16.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 485/3000 [00:25<01:55, 21.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▋        | 488/3000 [00:25<02:02, 20.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▋        | 491/3000 [00:25<01:56, 21.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 498/3000 [00:26<01:54, 21.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 505/3000 [00:26<01:59, 20.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 508/3000 [00:26<01:56, 21.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 514/3000 [00:27<02:08, 19.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 517/3000 [00:27<02:25, 17.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 524/3000 [00:27<02:06, 19.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 528/3000 [00:27<01:45, 23.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 531/3000 [00:28<02:27, 16.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 534/3000 [00:28<02:28, 16.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 540/3000 [00:28<02:18, 17.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 548/3000 [00:28<01:59, 20.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 551/3000 [00:29<02:05, 19.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▊        | 558/3000 [00:29<01:52, 21.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▊        | 561/3000 [00:29<02:16, 17.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 566/3000 [00:29<02:23, 16.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 572/3000 [00:30<02:01, 20.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 576/3000 [00:30<01:50, 21.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 579/3000 [00:30<02:04, 19.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 582/3000 [00:30<02:05, 19.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|█▉        | 588/3000 [00:30<01:57, 20.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|█▉        | 595/3000 [00:31<02:16, 17.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 598/3000 [00:31<02:17, 17.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|██        | 606/3000 [00:31<01:52, 21.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|██        | 613/3000 [00:32<01:54, 20.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 619/3000 [00:32<02:09, 18.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 625/3000 [00:32<01:39, 23.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 628/3000 [00:33<02:17, 17.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 634/3000 [00:33<02:00, 19.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██▏       | 640/3000 [00:33<02:02, 19.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██▏       | 643/3000 [00:33<02:05, 18.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 646/3000 [00:34<02:16, 17.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 653/3000 [00:34<01:49, 21.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 659/3000 [00:34<01:48, 21.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 662/3000 [00:35<02:41, 14.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 669/3000 [00:35<01:53, 20.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▎       | 675/3000 [00:35<01:56, 20.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 684/3000 [00:36<02:04, 18.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 687/3000 [00:36<02:00, 19.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 693/3000 [00:36<02:11, 17.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 698/3000 [00:36<01:41, 22.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 702/3000 [00:36<01:44, 21.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▎       | 708/3000 [00:37<01:56, 19.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 714/3000 [00:37<02:02, 18.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 718/3000 [00:37<01:54, 19.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 721/3000 [00:38<01:59, 19.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 727/3000 [00:38<02:04, 18.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 731/3000 [00:38<01:41, 22.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 734/3000 [00:38<01:50, 20.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▍       | 739/3000 [00:39<02:21, 15.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▍       | 747/3000 [00:39<01:39, 22.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▌       | 754/3000 [00:39<01:53, 19.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 757/3000 [00:39<01:55, 19.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▌       | 762/3000 [00:40<02:17, 16.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 769/3000 [00:40<01:44, 21.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 776/3000 [00:40<01:43, 21.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 779/3000 [00:41<02:00, 18.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 781/3000 [00:41<02:23, 15.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▋       | 788/3000 [00:41<01:48, 20.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▋       | 791/3000 [00:41<01:47, 20.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 796/3000 [00:42<02:09, 17.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 800/3000 [00:42<01:45, 20.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 803/3000 [00:42<01:53, 19.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 809/3000 [00:42<01:57, 18.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 812/3000 [00:42<01:50, 19.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 819/3000 [00:43<01:50, 19.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 822/3000 [00:43<01:59, 18.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 830/3000 [00:43<01:47, 20.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 833/3000 [00:43<01:51, 19.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 835/3000 [00:44<01:57, 18.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 840/3000 [00:44<02:01, 17.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 847/3000 [00:44<01:49, 19.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 851/3000 [00:44<01:34, 22.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 854/3000 [00:45<01:42, 20.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▊       | 859/3000 [00:45<02:09, 16.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 867/3000 [00:45<01:41, 20.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 870/3000 [00:45<01:46, 20.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 874/3000 [00:46<01:33, 22.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 877/3000 [00:46<01:51, 19.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|██▉       | 886/3000 [00:46<01:41, 20.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 891/3000 [00:46<01:45, 19.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|██▉       | 896/3000 [00:47<02:16, 15.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|███       | 902/3000 [00:47<01:44, 20.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 905/3000 [00:47<01:39, 20.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|███       | 911/3000 [00:48<01:46, 19.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 918/3000 [00:48<01:52, 18.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 922/3000 [00:48<01:41, 20.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 925/3000 [00:48<01:48, 19.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 931/3000 [00:49<01:55, 17.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 936/3000 [00:49<01:46, 19.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███▏      | 943/3000 [00:49<01:23, 24.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 946/3000 [00:49<01:42, 20.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 952/3000 [00:50<01:42, 19.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 957/3000 [00:50<01:59, 17.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 962/3000 [00:50<01:28, 23.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 965/3000 [00:50<01:48, 18.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 970/3000 [00:51<02:05, 16.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▎      | 975/3000 [00:51<01:32, 21.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 982/3000 [00:51<01:45, 19.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 985/3000 [00:52<01:42, 19.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 991/3000 [00:52<01:48, 18.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 995/3000 [00:52<01:41, 19.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 998/3000 [00:52<01:45, 18.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 1001/3000 [00:52<01:54, 17.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▎      | 1006/3000 [00:53<01:53, 17.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▎      | 1012/3000 [00:53<01:43, 19.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1017/3000 [00:53<01:27, 22.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 1023/3000 [00:54<01:33, 21.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 1029/3000 [00:54<01:58, 16.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1033/3000 [00:54<01:37, 20.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 1037/3000 [00:54<01:24, 23.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 1040/3000 [00:55<01:46, 18.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▍      | 1046/3000 [00:55<01:49, 17.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 1051/3000 [00:55<01:31, 21.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 1056/3000 [00:55<01:28, 21.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 1059/3000 [00:56<01:48, 17.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 1067/3000 [00:56<01:37, 19.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1070/3000 [00:56<01:31, 21.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1073/3000 [00:56<01:37, 19.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 1079/3000 [00:57<02:02, 15.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 1084/3000 [00:57<01:48, 17.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▋      | 1090/3000 [00:57<01:17, 24.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▋      | 1093/3000 [00:57<01:32, 20.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1096/3000 [00:58<02:01, 15.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 1104/3000 [00:58<01:28, 21.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 1110/3000 [00:58<01:19, 23.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1113/3000 [00:58<01:41, 18.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1116/3000 [00:59<02:00, 15.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1125/3000 [00:59<01:23, 22.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1131/3000 [00:59<01:18, 23.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1137/3000 [01:00<01:43, 17.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1140/3000 [01:00<01:38, 18.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1149/3000 [01:00<01:26, 21.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1154/3000 [01:01<01:47, 17.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▊      | 1158/3000 [01:01<01:47, 17.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▊      | 1162/3000 [01:01<01:28, 20.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1166/3000 [01:01<01:30, 20.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 1172/3000 [01:02<01:38, 18.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 1179/3000 [01:02<01:31, 19.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1182/3000 [01:02<01:36, 18.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|███▉      | 1187/3000 [01:02<01:45, 17.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 1190/3000 [01:03<01:40, 18.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|███▉      | 1195/3000 [01:03<01:52, 16.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1201/3000 [01:03<01:27, 20.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|████      | 1207/3000 [01:03<01:25, 21.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1210/3000 [01:04<01:31, 19.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|████      | 1215/3000 [01:04<01:58, 15.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1220/3000 [01:04<01:24, 21.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1223/3000 [01:04<01:39, 17.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1228/3000 [01:05<02:03, 14.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1233/3000 [01:05<01:53, 15.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████▏     | 1241/3000 [01:05<01:15, 23.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████▏     | 1244/3000 [01:06<02:20, 12.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1250/3000 [01:06<01:51, 15.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1254/3000 [01:06<01:36, 18.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1257/3000 [01:06<01:38, 17.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1262/3000 [01:07<01:53, 15.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1267/3000 [01:07<01:38, 17.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1272/3000 [01:07<01:27, 19.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▎     | 1275/3000 [01:07<01:21, 21.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1280/3000 [01:08<01:48, 15.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1282/3000 [01:08<01:42, 16.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1284/3000 [01:08<02:06, 13.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1289/3000 [01:08<01:50, 15.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1296/3000 [01:09<01:26, 19.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1301/3000 [01:09<01:47, 15.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▎     | 1308/3000 [01:10<01:39, 16.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▎     | 1311/3000 [01:10<01:37, 17.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1316/3000 [01:10<01:40, 16.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1321/3000 [01:10<01:25, 19.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1326/3000 [01:11<01:36, 17.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1332/3000 [01:11<01:24, 19.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1336/3000 [01:11<01:21, 20.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1340/3000 [01:11<01:17, 21.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1343/3000 [01:11<01:27, 19.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1346/3000 [01:12<01:44, 15.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▌     | 1356/3000 [01:12<01:19, 20.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1360/3000 [01:12<01:10, 23.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1363/3000 [01:13<01:24, 19.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1370/3000 [01:13<01:19, 20.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1373/3000 [01:13<01:15, 21.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1376/3000 [01:13<01:24, 19.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1384/3000 [01:14<01:21, 19.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1387/3000 [01:14<01:41, 15.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▋     | 1391/3000 [01:14<01:25, 18.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1398/3000 [01:15<01:30, 17.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1401/3000 [01:15<01:22, 19.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1407/3000 [01:15<01:27, 18.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1410/3000 [01:15<01:37, 16.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1415/3000 [01:15<01:19, 19.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1418/3000 [01:16<01:23, 18.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1424/3000 [01:16<01:25, 18.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1427/3000 [01:16<01:18, 20.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1430/3000 [01:16<01:24, 18.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1436/3000 [01:17<01:27, 17.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1440/3000 [01:17<01:09, 22.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1446/3000 [01:17<01:17, 20.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1451/3000 [01:17<00:59, 25.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1454/3000 [01:18<01:41, 15.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▊     | 1457/3000 [01:18<03:07,  8.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▊     | 1462/3000 [01:18<02:08, 12.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1466/3000 [01:19<01:44, 14.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1474/3000 [01:19<01:42, 14.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1477/3000 [01:19<01:33, 16.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1483/3000 [01:20<01:24, 17.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1486/3000 [01:20<01:22, 18.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|████▉     | 1492/3000 [01:20<01:24, 17.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|████▉     | 1498/3000 [01:20<01:23, 17.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1500/3000 [01:21<01:25, 17.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 1508/3000 [01:21<01:14, 20.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1511/3000 [01:21<01:10, 21.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1517/3000 [01:21<01:14, 19.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1520/3000 [01:22<01:13, 20.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1527/3000 [01:22<01:19, 18.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1531/3000 [01:22<01:04, 22.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████▏    | 1538/3000 [01:22<01:10, 20.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████▏    | 1541/3000 [01:23<01:12, 20.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1547/3000 [01:23<01:17, 18.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1553/3000 [01:23<01:14, 19.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1556/3000 [01:24<01:20, 17.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1564/3000 [01:24<01:06, 21.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1570/3000 [01:24<01:10, 20.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1573/3000 [01:24<01:19, 17.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▎    | 1575/3000 [01:25<01:21, 17.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1579/3000 [01:25<01:42, 13.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1585/3000 [01:25<01:02, 22.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1588/3000 [01:25<01:28, 16.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1593/3000 [01:26<01:14, 18.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1598/3000 [01:26<01:29, 15.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1603/3000 [01:26<01:14, 18.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▎    | 1611/3000 [01:26<01:01, 22.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1614/3000 [01:27<01:03, 21.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1617/3000 [01:27<01:27, 15.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1619/3000 [01:27<01:40, 13.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1625/3000 [01:27<01:12, 18.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1628/3000 [01:28<01:04, 21.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1634/3000 [01:28<01:24, 16.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▍    | 1639/3000 [01:28<01:17, 17.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1642/3000 [01:28<01:24, 16.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1646/3000 [01:29<01:05, 20.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1649/3000 [01:29<01:08, 19.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▌    | 1657/3000 [01:29<01:21, 16.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1661/3000 [01:30<01:16, 17.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1666/3000 [01:30<01:07, 19.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1669/3000 [01:30<01:09, 19.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1675/3000 [01:30<01:10, 18.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1681/3000 [01:31<01:07, 19.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1686/3000 [01:31<01:17, 17.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▋    | 1690/3000 [01:31<01:03, 20.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▋    | 1693/3000 [01:31<01:08, 19.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1696/3000 [01:31<01:20, 16.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1701/3000 [01:32<01:22, 15.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1703/3000 [01:32<01:28, 14.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1711/3000 [01:32<01:07, 19.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1717/3000 [01:33<01:15, 16.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1723/3000 [01:33<01:06, 19.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1726/3000 [01:33<01:01, 20.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1729/3000 [01:33<01:15, 16.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1734/3000 [01:34<01:26, 14.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1739/3000 [01:34<01:00, 20.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1743/3000 [01:34<00:53, 23.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1746/3000 [01:34<01:02, 20.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1751/3000 [01:35<01:17, 16.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▊    | 1757/3000 [01:35<01:03, 19.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▊    | 1760/3000 [01:35<00:59, 20.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1765/3000 [01:35<00:51, 23.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1771/3000 [01:36<01:21, 15.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1774/3000 [01:36<01:11, 17.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1781/3000 [01:36<00:51, 23.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1784/3000 [01:36<01:02, 19.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|█████▉    | 1790/3000 [01:37<01:18, 15.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|█████▉    | 1797/3000 [01:37<00:57, 20.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|██████    | 1803/3000 [01:38<01:01, 19.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1807/3000 [01:38<01:04, 18.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|██████    | 1812/3000 [01:38<01:14, 16.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1818/3000 [01:38<01:00, 19.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1822/3000 [01:39<00:50, 23.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1827/3000 [01:39<00:56, 20.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1832/3000 [01:39<01:24, 13.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████▏   | 1838/3000 [01:39<00:57, 20.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████▏   | 1844/3000 [01:40<00:57, 20.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1847/3000 [01:40<01:03, 18.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1852/3000 [01:40<01:08, 16.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1858/3000 [01:41<01:04, 17.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1862/3000 [01:41<00:57, 19.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1870/3000 [01:41<00:46, 24.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1873/3000 [01:41<01:03, 17.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1876/3000 [01:42<01:03, 17.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1879/3000 [01:42<01:06, 16.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1885/3000 [01:42<01:01, 18.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1889/3000 [01:42<00:51, 21.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1895/3000 [01:43<00:58, 18.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1898/3000 [01:43<01:05, 16.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1900/3000 [01:43<01:14, 14.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1903/3000 [01:43<01:10, 15.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▎   | 1910/3000 [01:43<00:53, 20.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1913/3000 [01:44<00:48, 22.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1916/3000 [01:44<01:06, 16.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1922/3000 [01:44<01:00, 17.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1926/3000 [01:44<00:52, 20.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1931/3000 [01:45<00:45, 23.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1934/3000 [01:45<00:58, 18.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▍   | 1942/3000 [01:45<00:53, 19.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 1950/3000 [01:46<00:48, 21.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1953/3000 [01:46<00:46, 22.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1956/3000 [01:46<01:09, 14.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 1961/3000 [01:46<01:04, 16.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1966/3000 [01:46<00:52, 19.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1969/3000 [01:47<01:01, 16.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1975/3000 [01:47<00:59, 17.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1981/3000 [01:47<00:46, 21.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1984/3000 [01:47<00:47, 21.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1987/3000 [01:48<01:08, 14.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▋   | 1990/3000 [01:48<01:16, 13.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 1996/3000 [01:48<00:51, 19.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2001/3000 [01:48<00:43, 23.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2005/3000 [01:49<00:58, 17.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 2013/3000 [01:49<00:52, 18.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 2020/3000 [01:50<00:48, 20.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2023/3000 [01:50<00:48, 20.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2026/3000 [01:50<00:53, 18.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2031/3000 [01:50<01:01, 15.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2036/3000 [01:51<00:56, 17.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2044/3000 [01:51<00:42, 22.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2047/3000 [01:51<00:58, 16.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2051/3000 [01:51<00:55, 17.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▊   | 2057/3000 [01:52<00:50, 18.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▊   | 2061/3000 [01:52<00:41, 22.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2064/3000 [01:52<00:48, 19.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 2071/3000 [01:52<00:52, 17.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 2077/3000 [01:53<00:43, 21.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2080/3000 [01:53<00:44, 20.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2083/3000 [01:53<00:57, 15.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 2086/3000 [01:53<00:49, 18.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|██████▉   | 2094/3000 [01:54<00:44, 20.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 2097/3000 [01:54<00:48, 18.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 2100/3000 [01:54<00:49, 18.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|███████   | 2106/3000 [01:54<00:52, 17.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 2109/3000 [01:55<00:53, 16.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 2117/3000 [01:55<00:40, 21.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 2123/3000 [01:55<00:39, 22.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2126/3000 [01:55<00:51, 17.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2128/3000 [01:56<00:52, 16.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 2134/3000 [01:56<00:42, 20.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████▏  | 2138/3000 [01:56<00:43, 19.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████▏  | 2144/3000 [01:56<00:38, 22.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2147/3000 [01:56<00:45, 18.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2152/3000 [01:57<00:47, 17.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2155/3000 [01:57<00:43, 19.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2161/3000 [01:57<00:45, 18.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2165/3000 [01:57<00:38, 21.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2170/3000 [01:58<00:59, 13.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2176/3000 [01:58<00:47, 17.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2181/3000 [01:58<00:38, 21.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2184/3000 [01:59<00:40, 19.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2187/3000 [01:59<00:54, 15.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 2195/3000 [01:59<00:38, 20.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      " {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2199/3000 [01:59<00:39, 20.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2202/3000 [01:59<00:39, 20.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▎  | 2205/3000 [02:00<00:45, 17.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▎  | 2207/3000 [02:00<00:57, 13.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 2217/3000 [02:00<00:34, 22.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2220/3000 [02:00<00:37, 20.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 2226/3000 [02:01<00:48, 15.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2231/3000 [02:01<00:44, 17.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▍  | 2239/3000 [02:01<00:31, 24.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2242/3000 [02:02<00:46, 16.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▍  | 2248/3000 [02:02<00:42, 17.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 2251/3000 [02:02<00:39, 18.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 2255/3000 [02:03<00:38, 19.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▌  | 2260/3000 [02:03<00:50, 14.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 2262/3000 [02:03<00:48, 15.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 2270/3000 [02:03<00:36, 19.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2273/3000 [02:04<00:33, 21.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2276/3000 [02:04<00:37, 19.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 2282/3000 [02:04<00:37, 19.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2285/3000 [02:04<00:34, 20.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▋  | 2292/3000 [02:05<00:33, 20.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▋  | 2295/3000 [02:05<00:49, 14.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 2302/3000 [02:05<00:38, 18.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 2308/3000 [02:05<00:34, 20.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 2314/3000 [02:06<00:43, 15.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 2321/3000 [02:06<00:31, 21.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2324/3000 [02:06<00:33, 20.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2327/3000 [02:07<00:36, 18.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2333/3000 [02:07<00:38, 17.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2335/3000 [02:07<00:42, 15.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2342/3000 [02:07<00:36, 18.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2345/3000 [02:08<00:33, 19.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2351/3000 [02:08<00:41, 15.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▊  | 2356/3000 [02:08<00:33, 19.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▊  | 2361/3000 [02:09<00:32, 19.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2364/3000 [02:09<00:40, 15.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2366/3000 [02:09<00:41, 15.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2369/3000 [02:09<00:37, 16.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 2374/3000 [02:09<00:39, 15.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 2381/3000 [02:10<00:27, 22.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2384/3000 [02:10<00:38, 16.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 2387/3000 [02:10<00:38, 15.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|███████▉  | 2391/3000 [02:11<00:49, 12.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|███████▉  | 2399/3000 [02:11<00:29, 20.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2402/3000 [02:11<00:45, 13.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2406/3000 [02:11<00:38, 15.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 2412/3000 [02:12<00:31, 18.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2418/3000 [02:12<00:27, 21.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2421/3000 [02:12<00:34, 16.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2424/3000 [02:12<00:30, 19.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2432/3000 [02:13<00:26, 21.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2435/3000 [02:13<00:34, 16.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████▏ | 2438/3000 [02:13<00:32, 17.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████▏ | 2444/3000 [02:13<00:31, 17.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2448/3000 [02:14<00:35, 15.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2454/3000 [02:14<00:25, 21.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2457/3000 [02:14<00:30, 18.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2459/3000 [02:14<00:30, 17.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2466/3000 [02:15<00:27, 19.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2471/3000 [02:15<00:30, 17.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▎ | 2475/3000 [02:15<00:27, 19.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2481/3000 [02:15<00:25, 20.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2487/3000 [02:16<00:31, 16.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2489/3000 [02:16<00:30, 16.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2495/3000 [02:16<00:29, 16.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2502/3000 [02:17<00:24, 20.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▎ | 2505/3000 [02:17<00:27, 18.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▎ | 2510/3000 [02:17<00:26, 18.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2514/3000 [02:17<00:29, 16.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2517/3000 [02:17<00:25, 18.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2523/3000 [02:18<00:21, 22.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2526/3000 [02:18<00:19, 23.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2532/3000 [02:18<00:25, 18.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2535/3000 [02:18<00:28, 16.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▍ | 2539/3000 [02:19<00:27, 16.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2543/3000 [02:19<00:22, 20.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2546/3000 [02:19<00:42, 10.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▌ | 2555/3000 [02:20<00:43, 10.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      " {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2558/3000 [02:21<00:35, 12.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2566/3000 [02:21<00:32, 13.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2573/3000 [02:21<00:26, 15.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2581/3000 [02:22<00:19, 21.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2584/3000 [02:22<00:21, 19.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2587/3000 [02:22<00:19, 20.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▋ | 2593/3000 [02:22<00:20, 20.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2596/3000 [02:23<00:25, 15.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2601/3000 [02:23<00:22, 17.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2603/3000 [02:23<00:23, 17.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2607/3000 [02:23<00:27, 14.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2613/3000 [02:24<00:19, 20.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2620/3000 [02:24<00:20, 18.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2625/3000 [02:24<00:21, 17.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2630/3000 [02:25<00:18, 19.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2633/3000 [02:25<00:20, 18.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2639/3000 [02:25<00:19, 18.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2643/3000 [02:25<00:18, 19.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2647/3000 [02:25<00:15, 22.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2653/3000 [02:26<00:18, 18.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2655/3000 [02:26<00:22, 15.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▊ | 2658/3000 [02:26<00:19, 17.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2666/3000 [02:27<00:17, 18.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2670/3000 [02:27<00:15, 21.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2673/3000 [02:27<00:16, 20.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2678/3000 [02:27<00:17, 18.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2684/3000 [02:28<00:18, 17.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|████████▉ | 2690/3000 [02:28<00:14, 21.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 2693/3000 [02:28<00:14, 21.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 2696/3000 [02:28<00:17, 17.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|█████████ | 2700/3000 [02:29<00:19, 15.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|█████████ | 2706/3000 [02:29<00:15, 18.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2709/3000 [02:29<00:16, 17.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|█████████ | 2715/3000 [02:29<00:15, 18.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2718/3000 [02:30<00:16, 17.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2720/3000 [02:30<00:16, 17.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2725/3000 [02:30<00:15, 17.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2728/3000 [02:30<00:15, 17.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2733/3000 [02:30<00:14, 17.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████▏| 2738/3000 [02:31<00:13, 19.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████▏| 2740/3000 [02:31<00:14, 17.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2745/3000 [02:31<00:15, 16.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2749/3000 [02:31<00:15, 15.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2758/3000 [02:32<00:08, 27.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2761/3000 [02:32<00:11, 20.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2764/3000 [02:32<00:12, 19.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2772/3000 [02:32<00:11, 20.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2778/3000 [02:33<00:10, 21.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2781/3000 [02:33<00:09, 23.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2787/3000 [02:33<00:12, 16.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2790/3000 [02:33<00:12, 16.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2797/3000 [02:34<00:09, 20.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2801/3000 [02:34<00:08, 24.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2804/3000 [02:34<00:12, 15.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▎| 2809/3000 [02:35<00:12, 15.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2817/3000 [02:35<00:07, 24.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2821/3000 [02:35<00:06, 26.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2831/3000 [02:36<00:07, 22.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▍| 2838/3000 [02:36<00:07, 21.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 2841/3000 [02:36<00:10, 15.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▍| 2849/3000 [02:37<00:07, 18.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2853/3000 [02:37<00:06, 22.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2857/3000 [02:37<00:08, 16.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▌| 2863/3000 [02:38<00:08, 15.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2871/3000 [02:38<00:06, 21.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2874/3000 [02:38<00:05, 21.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2877/3000 [02:38<00:07, 16.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2883/3000 [02:39<00:06, 18.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▋| 2889/3000 [02:39<00:06, 17.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▋| 2893/3000 [02:39<00:04, 21.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2896/3000 [02:39<00:05, 18.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2901/3000 [02:40<00:05, 16.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2903/3000 [02:40<00:06, 15.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2910/3000 [02:40<00:04, 19.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2917/3000 [02:40<00:04, 20.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2922/3000 [02:41<00:04, 18.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2927/3000 [02:41<00:04, 15.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2934/3000 [02:41<00:02, 22.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2937/3000 [02:42<00:03, 18.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2943/3000 [02:42<00:03, 18.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2946/3000 [02:42<00:02, 20.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2949/3000 [02:42<00:03, 16.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2955/3000 [02:43<00:02, 17.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▊| 2958/3000 [02:43<00:02, 19.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2964/3000 [02:43<00:02, 16.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2966/3000 [02:43<00:02, 12.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2968/3000 [02:44<00:02, 13.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2974/3000 [02:44<00:01, 19.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2977/3000 [02:44<00:01, 18.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2980/3000 [02:44<00:01, 19.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|█████████▉| 2985/3000 [02:44<00:01, 14.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|█████████▉| 2990/3000 [02:45<00:00, 15.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|█████████▉| 2998/3000 [02:45<00:00, 19.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|██████████| 3000/3000 [02:46<00:00, 18.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 09:52:20.045\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m<module>\u001b[0m:\u001b[36m4\u001b[0m - \u001b[1mEvaluation metrics (before optimization): {'f1': 0.0, 'em': 0.0, 'acc': 0.49733333333333335}\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "logger.info(\"Evaluating workflow on test set...\")\n",
    "with suppress_logger_info():\n",
    "    results = textgrad_optimizer.evaluate(dataset=benchmark, eval_mode=\"test\")\n",
    "logger.info(f\"Evaluation metrics (before optimization): {results}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "932092e0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'f1': 0.0, 'em': 0.0, 'acc': 0.49733333333333335}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "b59c472f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#generated_workflow\n",
    "qa_workflow = {\n",
    "    \"goal\": \"Provide a concise answer to the question using relevant context. The answer must be straightforward and avoid unnecessary explanations.\",\n",
    "    \"tasks\": [\n",
    "        {\n",
    "            \"name\": \"generate_answer\",\n",
    "            \"description\": \"Extract and formulate an answer from the given context.\",\n",
    "            \"inputs\": [\n",
    "                {\"name\": \"question\", \"type\": \"str\", \"required\": True, \"description\": \"The question that needs to be answered.\"},\n",
    "                {\"name\": \"context\", \"type\": \"str\", \"required\": True, \"description\": \"The background information pertinent to the question.\"}\n",
    "            ],\n",
    "            \"outputs\": [\n",
    "                {\"name\": \"answer\", \"type\": \"str\", \"required\": True, \"description\": \"The direct answer to the question.\"}\n",
    "            ],\n",
    "            \"prompt_template\": StringTemplate(instruction=\"Use the context to determine the best answer to the question. Provide your final answer in a clear format, without extra commentary or reasoning.\"),\n",
    "            \"parse_mode\": \"xml\"\n",
    "        }\n",
    "    ]\n",
    "}\n",
    "\n",
    "def collate_func(example: dict) -> dict:\n",
    "    problem = \"Question: {}\\n\\nAnswer:\".format(example[\"question\"])\n",
    "    return {\"question\": problem, 'context':''}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "2c62cea3",
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ[\"AZURE_OPENAI_DEPLOYMENT_NAME\"] = \"gpt-4o-mini\"\n",
    "os.environ[\"AZURE_OPENAI_ENDPOINT\"] = \"https://tianyuliu-hua-raredisea-resource.cognitiveservices.azure.com/\"\n",
    "os.environ[\"AZURE_OPENAI_KEY\"] = \"2pa9h2ZIN1lQepFWwYADlXIKIansa9KPhxMoumeGbRQ08f2uDTXiJQQJ99BKACHYHv6XJ3w3AAAAACOGsQIt\"\n",
    "os.environ[\"AZURE_OPENAI_API_VERSION\"] = \"2025-01-01-preview\"\n",
    "llm_config = LiteLLMConfig(model=\"azure/\" + os.getenv(\"AZURE_OPENAI_DEPLOYMENT_NAME\"),  # Azure model format\n",
    "    azure_endpoint=os.getenv(\"AZURE_OPENAI_ENDPOINT\"),\n",
    "    azure_key=os.getenv(\"AZURE_OPENAI_KEY\"),\n",
    "    api_version=os.getenv(\"AZURE_OPENAI_API_VERSION\", \"2024-12-01-preview\"), top_p=0.85, temperature=0.2, frequency_penalty=0.0, presence_penalty=0.0)\n",
    "\n",
    "executor_llm = LiteLLM(config=llm_config)\n",
    "optimizer_llm = LiteLLM(config=llm_config)\n",
    "llm = executor_llm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "d965a023",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "workflow_graph = SequentialWorkFlowGraph.from_dict(qa_workflow)\n",
    "agent_manager = AgentManager()\n",
    "agent_manager.add_agents_from_workflow(workflow_graph, executor_llm.config)\n",
    "\n",
    "evaluator = Evaluator(\n",
    "    llm=executor_llm, \n",
    "    agent_manager=agent_manager, \n",
    "    collate_func=collate_func, \n",
    "    num_workers=20, \n",
    "    verbose=True\n",
    ")\n",
    "\n",
    "textgrad_optimizer = TextGradOptimizer(\n",
    "    graph=workflow_graph, \n",
    "    optimize_mode=\"all\",\n",
    "    executor_llm=executor_llm, \n",
    "    optimizer_llm=optimizer_llm,\n",
    "    batch_size=3,\n",
    "    max_steps=20,\n",
    "    evaluator=evaluator,\n",
    "    eval_every_n_steps=1,\n",
    "    eval_rounds=1,\n",
    "    save_interval=None,\n",
    "    save_path=\"./\",\n",
    "    rollback=True,\n",
    "    constraints=[]\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "67291165",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3000"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(benchmark._test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "76b45e2f",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 09:53:37.184\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m<module>\u001b[0m:\u001b[36m1\u001b[0m - \u001b[1mEvaluating workflow on test set...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   0%|          | 7/3000 [00:00<04:54, 10.16it/s]Task exception was never retrieved\n",
      "future: <Task finished name='Task-45213' coro=<_client_async_logging_helper() done, defined at /gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/site-packages/litellm/utils.py:855> exception=RuntimeError('Event loop is closed')>\n",
      "Traceback (most recent call last):\n",
      "  File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/asyncio/tasks.py\", line 277, in __step\n",
      "    result = coro.send(None)\n",
      "             ^^^^^^^^^^^^^^^\n",
      "  File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/site-packages/litellm/utils.py\", line 873, in _client_async_logging_helper\n",
      "    GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(\n",
      "  File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/site-packages/litellm/litellm_core_utils/logging_worker.py\", line 322, in ensure_initialized_and_enqueue\n",
      "    self.enqueue(async_coroutine)\n",
      "  File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/site-packages/litellm/litellm_core_utils/logging_worker.py\", line 131, in enqueue\n",
      "    self._queue.put_nowait(task)\n",
      "  File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/asyncio/queues.py\", line 147, in put_nowait\n",
      "    self._wakeup_next(self._getters)\n",
      "  File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/asyncio/queues.py\", line 63, in _wakeup_next\n",
      "    waiter.set_result(None)\n",
      "  File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/asyncio/futures.py\", line 263, in set_result\n",
      "    self.__schedule_callbacks()\n",
      "  File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/asyncio/futures.py\", line 173, in __schedule_callbacks\n",
      "    self._loop.call_soon(callback, self, context=ctx)\n",
      "  File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/asyncio/base_events.py\", line 762, in call_soon\n",
      "    self._check_closed()\n",
      "  File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/asyncio/base_events.py\", line 520, in _check_closed\n",
      "    raise RuntimeError('Event loop is closed')\n",
      "RuntimeError: Event loop is closed\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 17/3000 [00:01<02:04, 23.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|          | 27/3000 [00:01<02:38, 18.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|          | 36/3000 [00:01<01:54, 25.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|▏         | 44/3000 [00:02<02:30, 19.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 52/3000 [00:02<01:58, 24.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 56/3000 [00:02<01:57, 25.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 60/3000 [00:03<01:47, 27.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 64/3000 [00:03<02:45, 17.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 76/3000 [00:03<01:49, 26.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 84/3000 [00:04<02:11, 22.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 94/3000 [00:04<01:41, 28.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 98/3000 [00:04<01:50, 26.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▎         | 106/3000 [00:05<02:04, 23.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 114/3000 [00:05<01:45, 27.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 117/3000 [00:05<02:04, 23.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 124/3000 [00:05<02:05, 22.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 129/3000 [00:06<01:59, 24.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 133/3000 [00:06<01:50, 25.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▍         | 141/3000 [00:06<01:57, 24.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      " {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▍         | 147/3000 [00:06<01:57, 24.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 153/3000 [00:06<01:31, 30.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 157/3000 [00:07<01:56, 24.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 160/3000 [00:07<01:52, 25.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 163/3000 [00:07<02:25, 19.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 173/3000 [00:07<01:38, 28.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 177/3000 [00:08<01:55, 24.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 181/3000 [00:08<02:31, 18.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 184/3000 [00:08<02:27, 19.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▋         | 191/3000 [00:08<02:47, 16.75it/s]Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6852e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5e1cb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6856580>, 2329380.773613764)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5e1fd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b7134a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee7c8390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b68562e0>, 2329380.652860819)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b750c710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6854c90>, 2329380.732868999)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b7136210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6856f20>, 2329380.623856131)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee7c9a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a15365c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b47ad350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5e35f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a15f5b690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b686a0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5ff25d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b60bb2a0>, 2329380.656304304)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b73c69d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b60b9da0>, 2329381.094575545)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b47ae190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6857c40>, 2329380.990900447)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5e35d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6856eb0>, 2329380.629211527)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a15f5b490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b68570e0>, 2329380.690694346)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6868550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6856270>, 2329380.712088349)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5ff2cd0>\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 198/3000 [00:09<02:33, 18.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      " {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 207/3000 [00:09<02:21, 19.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 217/3000 [00:10<01:58, 23.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 221/3000 [00:10<02:02, 22.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 224/3000 [00:10<01:59, 23.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 230/3000 [00:10<02:15, 20.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 236/3000 [00:11<02:09, 21.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 241/3000 [00:11<01:43, 26.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 244/3000 [00:11<01:57, 23.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 251/3000 [00:11<02:06, 21.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▊         | 258/3000 [00:11<01:47, 25.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▊         | 261/3000 [00:12<01:54, 23.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 268/3000 [01:00<2:13:44,  2.94s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 275/3000 [01:01<1:06:57,  1.47s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 283/3000 [01:01<31:27,  1.44it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 286/3000 [01:01<24:11,  1.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|▉         | 292/3000 [01:02<13:47,  3.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 295/3000 [01:02<10:52,  4.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 300/3000 [01:02<06:59,  6.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 303/3000 [01:02<05:48,  7.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|█         | 309/3000 [01:03<04:19, 10.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 317/3000 [01:03<02:40, 16.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 322/3000 [01:03<02:16, 19.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 325/3000 [01:03<02:49, 15.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 336/3000 [01:04<01:57, 22.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█▏        | 341/3000 [01:04<01:48, 24.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█▏        | 344/3000 [01:04<02:06, 21.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 350/3000 [01:04<02:02, 21.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 357/3000 [01:05<01:25, 30.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 361/3000 [01:05<02:00, 21.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 368/3000 [01:05<01:54, 22.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 371/3000 [01:05<01:52, 23.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 378/3000 [01:05<01:30, 28.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 382/3000 [01:06<01:42, 25.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 388/3000 [01:06<02:03, 21.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 395/3000 [01:06<01:44, 24.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 401/3000 [01:07<02:10, 19.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▎        | 410/3000 [01:07<01:36, 26.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 414/3000 [01:07<01:35, 27.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 417/3000 [01:07<02:12, 19.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 423/3000 [01:08<02:17, 18.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 428/3000 [01:08<01:44, 24.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 435/3000 [01:08<01:48, 23.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 438/3000 [01:08<01:44, 24.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▍        | 444/3000 [01:09<01:58, 21.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 451/3000 [01:09<01:45, 24.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 459/3000 [01:09<01:30, 28.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 462/3000 [01:09<02:03, 20.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 465/3000 [01:09<02:01, 20.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 474/3000 [01:10<02:04, 20.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 479/3000 [01:10<01:40, 24.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▋        | 488/3000 [01:11<01:49, 23.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▋        | 493/3000 [01:11<01:44, 23.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 496/3000 [01:11<02:06, 19.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 505/3000 [01:11<01:46, 23.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 512/3000 [01:11<01:30, 27.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 516/3000 [01:12<01:48, 22.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 522/3000 [01:12<01:55, 21.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 528/3000 [01:12<01:25, 28.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 533/3000 [01:13<03:21, 12.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 534/3000 [02:00<2:46:18,  4.05s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 536/3000 [02:01<2:14:51,  3.28s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 543/3000 [02:01<1:02:15,  1.52s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 546/3000 [02:01<45:23,  1.11s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 553/3000 [02:02<22:51,  1.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▊        | 560/3000 [02:02<12:27,  3.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      " {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 567/3000 [02:02<07:47,  5.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 570/3000 [02:03<06:51,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 581/3000 [02:03<03:14, 12.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 585/3000 [02:03<03:20, 12.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 589/3000 [02:03<02:58, 13.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|█▉        | 598/3000 [02:04<01:55, 20.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 602/3000 [02:04<02:01, 19.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 605/3000 [02:04<02:04, 19.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|██        | 611/3000 [02:04<01:59, 19.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 618/3000 [02:05<01:37, 24.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 621/3000 [02:05<01:46, 22.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      " {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 629/3000 [02:05<01:36, 24.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 635/3000 [02:05<01:33, 25.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██▏       | 641/3000 [02:05<01:35, 24.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██▏       | 644/3000 [02:06<01:30, 25.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 650/3000 [02:06<01:51, 21.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 655/3000 [02:06<01:38, 23.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 658/3000 [02:06<01:55, 20.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 662/3000 [02:06<01:39, 23.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 671/3000 [02:07<01:47, 21.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 677/3000 [02:07<01:45, 21.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 680/3000 [02:07<01:51, 20.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 686/3000 [02:08<01:37, 23.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 692/3000 [02:08<02:01, 19.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b514a150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b692e7d0>\n",
      "Evaluating workflow:  23%|██▎       | 695/3000 [02:08<03:15, 11.79it/s]Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5ad7a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4971f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4ef0ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b7330050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b60b9be0>, 2329380.780804795)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b7332c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b60b9860>, 2329380.71084237)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b514a890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6854520>, 2329380.853598557)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a15645b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6857620>, 2329380.761616557)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5ad7f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6854440>, 2329381.10639073)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4972e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b68542f0>, 2329380.756386062)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4ef3bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b67e11d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b60b9a90>, 2329380.601480969)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b67e2e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee60ec10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b60b9780>, 2329380.718464456)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee60cc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b71363d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee9a8050>, 2329381.612072981)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b7135b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b640db10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6b5ad60>, 2329381.418332547)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5de4150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b70e1250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150b1e496a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7087850>, 2329381.670698469)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6f65ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7086ac0>, 2329381.850732692)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b674b350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b607d450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b70864a0>, 2329381.567546099)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b607c390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee60c790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b69b4310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b7402350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee7a9f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5c58c20>, 2329381.773091948)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b70dd490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b62b2820>, 2329381.871249242)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee7ab390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b70e5470>, 2329382.824697288)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5a41790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b63c4440>, 2329381.7618284)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b7403e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6f1a510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b70e4280>, 2329382.753055645)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6f18e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b67e1ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b70e5cc0>, 2329382.549786978)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b66f2190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee90e950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6b5bbd0>, 2329383.135991343)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5e87b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5840850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b639de80>, 2329383.246818432)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5842690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6f0e190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6eb2f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b647f3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b658b490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b70e6dd0>, 2329382.532592386)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6eb2950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b62b3460>, 2329382.608239653)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b647d290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b70e6eb0>, 2329382.423820769)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6f0df10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b70e4670>, 2329382.708359432)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b658b090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6811b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b639d1d0>, 2329383.543025858)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5d19e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5b054d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5e5f350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b710b8c0>, 2329383.239380108)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b69fa150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b710a900>, 2329383.273368386)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b633e290>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5b31b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b710b380>, 2329383.446719745)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5b31450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5f3f890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b710a970>, 2329383.615908616)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5f3e510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b52bcf10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b70e6ba0>, 2329383.935133384)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b52be3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6357fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a14fe36d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b59c5b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6156cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b639d710>, 2329383.294765826)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee693d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b639d550>, 2329383.336299225)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a14fe3e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6edead0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b73fbe50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b710a0b0>, 2329383.369293271)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6edea50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7108a60>, 2329383.349403765)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6154f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b62b2970>, 2329383.360228839)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b73fb410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b71083d0>, 2329383.418855196)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6475d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5a9f890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b623d310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7208830>, 2329384.167077464)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5a9f110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7209860>, 2329384.16088939)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b64f8910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b75218d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b639e820>, 2329384.02715133)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b7520f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee5fa310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4563310>, 2329384.078029417)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee5f8690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b7563250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b639c750>, 2329384.343340354)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b7560c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5509750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee7a8d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b649ced0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b526f410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b62b2740>, 2329383.985132912)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4849190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b65b11d0>, 2329384.175944279)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b550b310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7209780>, 2329384.074227041)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b649e250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7208fa0>, 2329384.210764081)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b526da90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6b24150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4e8ad60>, 2329384.700981687)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6b25d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5bfc750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4fd2d60>, 2329385.024579639)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6655190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4af5050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4fd38c0>, 2329384.987621019)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5aa53d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b52a4150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5803b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4827d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b691d4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7209320>, 2329384.708219194)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b7303c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b70e7070>, 2329384.69529935)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b691f5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4e89940>, 2329384.668438174)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b48266d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4e8b770>, 2329385.447974777)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a153c7590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4af68d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b70e65f0>, 2329384.745162373)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4af6c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6249590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4fd1400>, 2329385.307972811)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b624b1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5bf2110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4951be0>, 2329385.848449036)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee62dc10>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4788d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4951010>, 2329386.319849802)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4789310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b52ad790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6ecad90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5537410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4951470>, 2329385.95288757)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6ec9950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4953460>, 2329386.021394549)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5535890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4f94750>, 2329386.541332586)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b52adc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b642a190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4f967b0>, 2329386.398844574)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee9200d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a15af6d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b61bf3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b506de80>, 2329386.92931504)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee6498d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b506e040>, 2329386.944657147)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b61bdf50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6ec9050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee69f750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee7e3e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee849150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5f8e110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4f94f30>, 2329386.589441643)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee849dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4f96cf0>, 2329386.612324537)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee69dc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b46756a0>, 2329387.184095423)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5965110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b725f990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4f95400>, 2329386.693185195)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b721e7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4f96820>, 2329386.585420949)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5f8eb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4f977e0>, 2329386.569194199)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b651d810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a155133d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453e120>, 2329387.26577572)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a155138d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a15642b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b506cc90>, 2329387.52878922)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b72536d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee6497d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee6b0710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b506f5b0>, 2329387.30972641)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee6b3d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453cd70>, 2329387.807809118)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b754e650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee611690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5addbe0>, 2329388.331310765)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee613350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5547510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4fd3e70>, 2329388.260660363)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b55468d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b60f1d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453e510>, 2329389.447408249)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b60f0b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b572fe90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b626d1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a15037690>, 2329387.986845018)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b626fc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453d7f0>, 2329387.756105307)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b572dfd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b57e84d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a15035cc0>, 2329389.105465838)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b57e8610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b485ea10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5ea8440>, 2329389.190956081)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b485d010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a1549c3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b45dd290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5ea90f0>, 2329389.100228227)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a14fdfe90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5eab690>, 2329389.102797599)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b45dc650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b59c2850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5eaa6d0>, 2329389.104278634)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b59c3590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b48f4290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b506dda0>, 2329389.145978264)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b48f76d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b53be450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b53bcf50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5eaa510>, 2329389.51672238)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5185090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b493ea10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b45537d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b48adc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b58b66d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee643090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b45f3650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b63ac290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b506e350>, 2329388.768407647)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b59e1e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5ea8b40>, 2329388.770369598)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee642910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b49511d0>, 2329388.765717698)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150b15fdbed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5adc520>, 2329388.77313765)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b45f2c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5adf7e0>, 2329389.603472868)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee652cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a155212b0>, 2329389.393148192)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b493e410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5add860>, 2329388.77543578)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b58b61d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6e5dd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a153e9090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b47a2390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5ad6d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b686ba50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b7396ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b66868d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b75ae190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b683afd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5bf8450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5a41010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee683110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b489cc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6b0d650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5e8c210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5d39ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee83cf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6474290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b7309990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5da9350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5fea0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6e53fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee5536d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b608ce10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6594f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6813690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6714f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6832450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b47ae3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee7aa750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b57ece10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b623c6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6719b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5fadd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b60e92d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee5fac50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b7073990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5c3af50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b588d6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b691e8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b74fb810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5335210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b7562750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b57cc850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4777b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee95a750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4a92c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5344f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b598cdd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee922b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5d6ec90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b61ca390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5f87d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a153e0210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b52a7890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5b73d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b605e590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee959fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b7211ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a1553eb50>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b61c9f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5f9db90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b738b1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b57cf410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b75d3890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6b9ec90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6fa9850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5867590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b639a4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee7e1d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b58fbc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5df2f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b488dc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b602f750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b75e9590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee902f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a15518ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5d79650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b559a850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b668e310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee874c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee7b5f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b52fb7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6b20c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a1562d5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee68a3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a1551a110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b51dbf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6520290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b60757d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b559b410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b47641d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6336ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b64d7a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee787f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5b77350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b60cdd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4764950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee6515d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b60805d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b58a0f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6242b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b48d3850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5ffe1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b57c6c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b579cd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a15035a90>, 2329388.764168103)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b62424d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a150373f0>, 2329388.783732556)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4766f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a150354e0>, 2329389.51552215)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4553d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee9a92b0>, 2329390.508583955)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6e5c2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b60bb540>, 2329380.702585475)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a153e9b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6855470>, 2329380.64272724)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b47a28d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6279e80>, 2329381.484041075)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5ad4150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a159bb5b0>, 2329381.438453373)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6868510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b63c6d60>, 2329381.500346913)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5d9e890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6b5a7b0>, 2329381.599671963)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6685e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6b59550>, 2329381.859847895)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b490d2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6b59be0>, 2329382.168088882)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6838050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7086200>, 2329381.642154806)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5bf8d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7087af0>, 2329381.50872987)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5a42f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b60ba6d0>, 2329381.496015381)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee682a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6b588a0>, 2329381.699669142)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5b36090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b63c4210>, 2329381.691846456)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6b0d1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7086dd0>, 2329381.582922398)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5e8d3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b62b2900>, 2329382.156464987)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5d38090>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b62b0130>, 2329382.314018461)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b698ad50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b62b0830>, 2329382.419272037)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6474650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b70e69e0>, 2329382.490360147)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b730b090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7115fd0>, 2329382.359609348)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5daa350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b70e6820>, 2329382.565490436)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5fe9cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b70e6d60>, 2329382.545414333)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6e51010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a159bb7e0>, 2329382.144405957)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee553c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b62b2430>, 2329382.323519784)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b608d010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b62b1c50>, 2329382.385200513)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6594e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b70e5320>, 2329382.55169914)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6810e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b70e48a0>, 2329382.788324909)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b75ac050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b639c1a0>, 2329383.27541415)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6530350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b639d0f0>, 2329382.875815922)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6563910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b639d780>, 2329383.134213268)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b68132d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b70e7bd0>, 2329382.782725045)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b57edf90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b70e7b60>, 2329383.282768948)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b623ce90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b71089f0>, 2329383.907769477)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b671a410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b710b070>, 2329383.514205521)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5faf1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7109160>, 2329383.687679743)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b60e9c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b710b930>, 2329383.905853359)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee5fb650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b70e5780>, 2329383.858971398)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5329750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b710be70>, 2329384.201006495)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a15adb6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4562f20>, 2329384.271007552)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b588d590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7108ad0>, 2329384.444819364)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b691c5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b710a510>, 2329384.249669488)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5509610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4560670>, 2329384.961305469)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5336710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b710bc40>, 2329384.460143891)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b484b9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b45612b0>, 2329384.439650454)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b640df90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b70852b0>, 2329384.764875104)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4775a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4e8add0>, 2329385.014054807)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee959690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4e8bd20>, 2329384.769460522)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b48a3450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4e8aac0>, 2329384.862179041)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5346290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4561860>, 2329384.925019939)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b598fc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4fd29e0>, 2329385.033226309)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee920610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4fd0980>, 2329385.113966221)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b60898d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b45626d0>, 2329385.068505622)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b49b51d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4fd0050>, 2329385.191171491)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5f856d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4fd3cb0>, 2329385.279476612)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a153e1450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4676430>, 2329385.636222253)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6f91250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4675710>, 2329385.502795801)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5b71e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4675da0>, 2329385.442910892)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b605f990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4677d20>, 2329385.592749001)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee959810>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4674830>, 2329385.537522901)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b75c7d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4675780>, 2329385.891459796)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a1553e550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b46741a0>, 2329385.950633116)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b61c9f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b46757f0>, 2329385.873795993)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b49b4350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7209cc0>, 2329385.729717908)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b7388a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4675a20>, 2329385.722021625)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee802c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4951ef0>, 2329385.825998677)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b75d0d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4e89400>, 2329385.556380266)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b7493cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4674f30>, 2329385.700396171)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6faa8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4fd06e0>, 2329385.718463188)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5866250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4fd0830>, 2329386.196914374)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b639acd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4674590>, 2329386.195704986)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee7e3910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b45632a0>, 2329386.253718121)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b58fb5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b710a660>, 2329386.414445573)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5df13d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4f969e0>, 2329386.458758831)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b476ced0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4f97690>, 2329386.832510551)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b602de10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4560ec0>, 2329386.017496144)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b478b6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b639f070>, 2329386.254891187)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b72d0490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4677c40>, 2329386.567998165)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a155185d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4f941a0>, 2329386.502553272)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5d7b650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b506e820>, 2329386.6657735)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b716dc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b506cb40>, 2329386.948990257)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b668cad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b506fcb0>, 2329386.803610725)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee8770d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b506f7e0>, 2329387.050422587)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee7b6910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b506fe70>, 2329386.992244136)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b52f8bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b506c590>, 2329387.542018124)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6b232d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453f690>, 2329387.489983067)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a1562e5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453c1a0>, 2329387.421654653)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee689650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453fd90>, 2329387.452258791)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b7490b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453fb60>, 2329387.543616548)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee70cf50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453e580>, 2329387.580172434)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6520310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453dc50>, 2329387.476766664)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6075890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4f95470>, 2329387.575063305)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6235010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b506e900>, 2329387.671233631)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4766710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a15034360>, 2329387.769490002)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b593bc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a15036040>, 2329388.147511963)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b64d7b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453e0b0>, 2329387.498609884)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee787210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4f94ec0>, 2329387.879155432)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5b76390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b506eba0>, 2329388.196892032)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b60cf750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5ade2e0>, 2329388.337189054)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee653ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5ade6d0>, 2329388.327365286)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a15641a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5adef90>, 2329388.370663506)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b58a2650>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4951160>, 2329388.152392779)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b48d1d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4676510>, 2329388.295879878)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5ffd710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b49529e0>, 2329388.359844817)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b63ea010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5adfee0>, 2329388.363842005)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b579c650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5342c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b63af790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4e89cc0>, 2329390.23467066)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b73b7f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4f95550>, 2329389.939404384)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b69c0d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5df3450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453cbb0>, 2329389.959261337)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b54c2250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b7555490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b71cd5c0>, 2329390.475173447)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b7439a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4848b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b583db70>, 2329390.364672059)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b484a090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6f2d8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453c7c0>, 2329390.250036024)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6f2c0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6fd7d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a15034c20>, 2329390.618892064)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6fd70d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee9e1010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a14fd0250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4ee49d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b470f590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4ae0590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4800690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4801850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6064090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b49631d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4ac9990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b657d710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5750fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b526d610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4e8b1c0>, 2329389.898631758)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b69c3f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453e740>, 2329390.044452528)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4ac9d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a16164ad0>, 2329389.868003981)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b657ded0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4953ee0>, 2329389.890861797)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4709190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4951b70>, 2329389.921145996)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee6854d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b71ccec0>, 2329389.997884944)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5751410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4953070>, 2329390.13288288)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4888e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5d64ec0>, 2329390.55443486)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee9e0310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4675fd0>, 2329390.604840253)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a14fd0350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b720a510>, 2329389.875106141)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b535d750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4950ec0>, 2329389.791561007)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee918a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453d940>, 2329389.834742555)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4962950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5d64590>, 2329390.209785872)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b526f8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4786910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453e270>, 2329389.862177648)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4786f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a15443ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5df6120>, 2329390.897658118)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a15441f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b75d1d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5df4d70>, 2329390.885099032)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b75d3bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b457f210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7001e10>, 2329390.88048153)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b457dd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b55454d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7002f20>, 2329391.004139527)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b673dc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b68339d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b63d86d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7000b40>, 2329390.995155338)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a15384990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4ae3ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5df7a10>, 2329390.965236765)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4ae1b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5007450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b71b54d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7193af0>, 2329391.06436741)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5004250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee68e580>, 2329391.166550325)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b57c5790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5a0e410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b639cad0>, 2329391.180209581)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee98f650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b569d990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee63d690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b483e350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee68d550>, 2329390.898814189)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee6c3b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7193150>, 2329391.24144171)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5a0f350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b515e9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5df7b60>, 2329390.734264508)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b45eb6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee68e120>, 2329390.732882262)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b483db50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5df47c0>, 2329390.703700651)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b45519d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5df42f0>, 2329390.683294322)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee63e750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b53be190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5df7460>, 2329390.781944711)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b53bdf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6b0cc50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4add350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b67f0280>, 2329391.7181782)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4982310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b583d940>, 2329391.547085381)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4adde50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4a68950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b67f3bd0>, 2329391.482538232)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4a68050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6301090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b481e610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7192ba0>, 2329391.487869342)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b71b4750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4ae3f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b7402250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b68df620>, 2329391.573915856)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b49be790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b68dec80>, 2329391.683270972)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b7402490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6e41810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b68dfa80>, 2329391.608538418)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6e43f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a15015c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee68dfd0>, 2329391.789008332)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b647f1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a15015b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b744f1c0>, 2329391.935261621)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5466cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6380f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b514a110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5c79a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b744cde0>, 2329391.959198095)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b608dc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b744faf0>, 2329391.741080822)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5149450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b70038c0>, 2329391.845849628)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a154c5c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a15637150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b744c600>, 2329391.805443449)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a15636f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5ff3250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee778ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b68de2e0>, 2329391.430772608)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6300c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a15414c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b744d9b0>, 2329391.750044991)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5ff28d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b67f07c0>, 2329391.938954808)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee77b150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b68de200>, 2329391.515828747)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a15414ad0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a15493fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6312350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5911be0>, 2329440.362709704)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b68b5490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5a84ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5bd1690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5912740>, 2329441.243262103)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5bd0b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6594690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5912270>, 2329440.366470398)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6595710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5c7af10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b69a3e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee7f3f50>, 2329440.495963066)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5c794d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b68de890>, 2329440.455738431)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b69a0b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4adf5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee7f0590>, 2329441.403695089)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4add7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b70dc450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b67f0fa0>, 2329442.038184392)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6f1bfd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b652ebd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5be9c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5f82190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b68ddbe0>, 2329441.380255822)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a15490410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5fb80c0>, 2329440.376787215)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5a87bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee779610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b77f7a10>, 2329441.590472453)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b652c0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b77f6c80>, 2329441.985540739)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5bea910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5fbb540>, 2329441.349548369)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5f82b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b68deb30>, 2329441.451066561)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee77a550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b71c8e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5fb8210>, 2329441.33275225)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b71ca390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6edb690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b746e820>, 2329441.551403823)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6ed8250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5a36990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7115390>, 2329441.479654274)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4ae3f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6997390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6509f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7115010>, 2329441.50836923)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6994a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b71173f0>, 2329441.399956419)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6509d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a154ada10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7117cb0>, 2329441.283683159)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a154ade90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a153cbf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5e07750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5c7f690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6513090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b67261d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b583c980>, 2329440.930241002)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6727fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7117150>, 2329441.236132029)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a15365c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7114600>, 2329441.883750517)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b55f3d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5edaf90>, 2329441.785526297)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5c7dbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5ed98d0>, 2329441.859816455)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6513e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5d331d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b61c5c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5eda890>, 2329441.727721177)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5d32050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b746c520>, 2329440.734337088)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b61c5ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b49ebe50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b67f2eb0>, 2329442.389785289)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b49ea890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee86ad50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7114e50>, 2329442.447926345)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee86a510>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5c42b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b75a4990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4564ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6854c20>, 2329442.301175728)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b75a7390>\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b67f33f0>, 2329442.329027977)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4567e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b514e610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b581aad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee7f2510>, 2329442.356363792)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b581a6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5dc6890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5a5e690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6854d70>, 2329442.24392101)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b732fd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b68572a0>, 2329442.33958507)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5a5c790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a1535dd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4889590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b67bf590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b67252d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b71174d0>, 2329443.196883115)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a162282d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b60b9e10>, 2329441.725563636)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b488ad90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6857310>, 2329442.978231842)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6b08450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5910280>, 2329442.461899954)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a1535e9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee7bd150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b60bb850>, 2329442.070233755)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5218290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee7f1f60>, 2329441.961048906)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6726990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b60b8590>, 2329442.185401634)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee7bcf50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b76bffd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee9a8b40>, 2329442.852748053)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b76bdcd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b62c0050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a15375610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b60b8c90>, 2329443.037185709)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee6fdc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b60b9710>, 2329443.25290654)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6473e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5859610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7116970>, 2329442.841054267)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b585bb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b66a3dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b76f2f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7193f50>, 2329442.950113831)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b66a1f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b65b0980>, 2329443.081830505)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b76f1710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee984690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6854980>, 2329443.060764349)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee984590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b713e5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6857850>, 2329443.046683762)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5313390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a15376d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4dfdf90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b65b1470>, 2329444.358112895)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b59fdad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b65b0d00>, 2329443.091909392)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4dfc690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6ee4b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b56afa50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6f1c190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b73c5ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b667b250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b57f4390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6855a20>, 2329442.442115665)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b57f44d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b68579a0>, 2329442.601079827)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6f1ca90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6855010>, 2329442.64380447)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b73c6e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b60bb4d0>, 2329443.062050222)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6ee7050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b59137e0>, 2329443.206093626)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b56aead0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b60b89f0>, 2329442.770296557)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6679910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b67e1dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b756f9a0>, 2329443.223947073)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b67e37d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b51c7a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5e26190>, 2329443.745879609)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b51c7a50>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee9ae3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b68562e0>, 2329443.435775584)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee9ac090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5639e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4707350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5e24b40>, 2329443.689562708)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5639290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b60b9fd0>, 2329443.681624438)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4705790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b56dcb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee9dc590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee9283d0>, 2329443.722436572)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee9de610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b494e510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b58fe610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee929080>, 2329443.884398034)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b494f5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6f65e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee515610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4a637d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5252750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6565190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5b81c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5ed9be0>, 2329443.767506147)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b56de490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b756cfa0>, 2329443.202870005)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5252090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b756de80>, 2329443.276063721)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6565e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b65b26d0>, 2329444.14663076)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b58fe050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b65b23c0>, 2329443.387151237)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5b82b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee92a900>, 2329443.903018438)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6f65750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5edb540>, 2329443.924151563)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee5179d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee92b2a0>, 2329443.926142052)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4a62ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b550e7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee928f30>, 2329444.064773864)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b550db50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b494a850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee928050>, 2329444.234893179)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b494b910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b58fe110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5453f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5566270>, 2329444.572893088)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b58ffa10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a1503b890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5565390>, 2329444.448283027)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a1503b2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b565c6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b756d400>, 2329444.620279659)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a150a7a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b494aad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4e66310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b558b450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a14fcc510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee7753d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee7dc250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee929400>, 2329443.946296384)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b558b510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee92a190>, 2329443.901812419)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a150a1d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5e25390>, 2329443.919707011)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee775a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee92aeb0>, 2329444.828858878)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5453e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b58fc6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee928210>, 2329444.228966281)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4d37f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5567620>, 2329444.535276437)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b494a590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee92b1c0>, 2329444.069963027)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a14fcc6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee929da0>, 2329444.062599548)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b45054d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee61e6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b63c6900>, 2329444.109387429)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee61c710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b54ae990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5c5aac0>, 2329444.74855077)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b54cb890>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4996490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5c5a900>, 2329444.738021559)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b568d810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee775190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5c597f0>, 2329444.636578813)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee776650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b59db050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b54cac10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5995790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5c58360>, 2329444.807351896)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b54ca5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5c5ab30>, 2329444.872799834)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5995e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b69e1b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5c5a270>, 2329444.838560534)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b69e1950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a150a45d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee928750>, 2329445.022318395)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a150a4950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6998910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee92bee0>, 2329445.179851813)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6998e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b54c8050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4562660>, 2329445.262339654)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b699a850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6094c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b63c4d70>, 2329445.163029207)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b601b6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b60ded50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4907210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b49db2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b59b05d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5c5be70>, 2329444.892677679)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a14fcf390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5c58520>, 2329444.746122103)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b49db090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5c59390>, 2329444.729155791)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b59b0b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b72b0050>, 2329445.269173626)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b60deb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b63c4de0>, 2329444.718948529)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b65e5190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee775d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b63c6820>, 2329444.820904522)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b54065d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b694d350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6278750>, 2329445.351396501)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4ddaed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4907950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4874b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b72b22e0>, 2329445.737535087)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4904c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b627a820>, 2329445.583554635)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4875090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4876b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6278f30>, 2329445.445432889)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4876a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b48cf590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6094c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b48d7950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b627b5b0>, 2329445.602742123)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b48cc110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b65b3af0>, 2329445.658395346)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b694e350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4a81410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b627a120>, 2329445.52957883)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4a81b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b49486d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5eab380>, 2329445.791443108)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a14ff4c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4a1bd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4adac50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5eaac80>, 2329445.626325083)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4adb2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4aaf210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5ea9010>, 2329445.606623583)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4aad390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b60dffd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4d27790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4dc4950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4dd9650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7116040>, 2329445.565778828)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b48d60d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5eab540>, 2329445.647439691)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4a1bf10>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5c58440>, 2329445.300759353)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4d25890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5ea82f0>, 2329445.745924864)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4da82d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5c5a350>, 2329445.363923924)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4d7b6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b627b3f0>, 2329445.527254675)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee7dee90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b47950d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5ade120>, 2329446.026982033)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4794f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b48cbc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a15520b40>, 2329445.941432341)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4de68d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b58fc250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b47f2490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5ea9550>, 2329446.173212763)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b47ec910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5e26820>, 2329446.108658961)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b47f2c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4652310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b46c78d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a15521b00>, 2329446.299800041)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b46c7790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b46eda10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b72b02f0>, 2329446.312185134)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b46ed8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b585b6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4684110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a155225f0>, 2329446.427052025)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a150a4e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a15522a50>, 2329446.537101499)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4684490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b463b6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6278c90>, 2329446.398720821)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b463b010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4632650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b44c53d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5c59470>, 2329446.428237751)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4631910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b44c6b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b44cd210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a155239a0>, 2329446.569106732)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b44c7190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b627bf50>, 2329446.52120434)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b44cd910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b44aa990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a15521940>, 2329446.21519541)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b46c2d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b47d9750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a15520050>, 2329446.327033154)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b44c5850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5adf310>, 2329445.934344722)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b47d9550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b446c600>, 2329447.086158083)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b44aa790>\n",
      "Evaluating workflow:  24%|██▎       | 705/3000 [02:09<01:57, 19.48it/s]Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4a7d6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b446c7c0>, 2329446.389403689)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4a7f050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4480310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b446d1d0>, 2329446.812136319)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4480810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4402f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b446e660>, 2329446.903954734)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4402dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b440c4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee929e80>, 2329446.95854598)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5343290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b46516d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b446f700>, 2329447.029236607)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4119990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b44c7950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5adecf0>, 2329446.83709277)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b44ac110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4780c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b415bc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b41502f0>, 2329447.141025233)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4139cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b41508a0>, 2329446.94246628)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b415bad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b416df90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b41511d0>, 2329447.04112956)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b47e6410>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b44ce590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b418d4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b63c5240>, 2329447.030665627)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4de7bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b41515c0>, 2329447.180745523)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b418c6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b41a09d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4422550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b44734d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5adc590>, 2329446.954235605)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b47db6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b446e970>, 2329447.225495104)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4400c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b446d7f0>, 2329446.892379393)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b44732d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4138810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4151ef0>, 2329447.205974261)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4420f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b412ca90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b72b26d0>, 2329447.151602667)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b412cbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b41dc410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b41500c0>, 2329447.47885386)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b41db9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b41f45d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f931090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6278910>, 2329447.611776722)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f931790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4dda450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f918b40>, 2329448.284730181)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f915d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f939350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f9196a0>, 2329448.041855301)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f93a050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f903f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f96ed90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f919860>, 2329447.94616375)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f903ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6278c20>, 2329447.65934754)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f96f310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b41c9350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f9826d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f91a270>, 2329448.30019574)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f95a410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f91a430>, 2329447.850038243)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f9824d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4160dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b41d8b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4153850>, 2329447.41288359)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b41f4450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4100290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4152900>, 2329447.298678241)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b41003d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f91b000>, 2329448.39734962)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4160790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f91ae40>, 2329447.909155225)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b41d91d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b44ae650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a15523310>, 2329447.323964922)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4421a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4601090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f91b1c0>, 2329448.057512387)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f9207d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f94a6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4153540>, 2329447.973637036)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f94a710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4480c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4153150>, 2329448.262341491)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4480290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f8582d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f87bfd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f919fd0>, 2329448.20559649)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f858150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f88ead0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f9ef700>, 2329448.348143112)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f88e8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f89f290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f9efe00>, 2329448.827569202)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f89f150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f9c08d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f9f1fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f80f310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f817350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f850f90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f918600>, 2329448.412025526)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f87bed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b446cd00>, 2329447.968305977)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f817150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f9e6290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b446c0c0>, 2329448.072151724)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f9f2610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4151160>, 2329448.013456373)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f9e6550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f9187c0>, 2329448.506220398)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f80dc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f9df210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f91b850>, 2329447.716044757)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5250e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f919b70>, 2329448.256743377)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f850d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f9ec6e0>, 2329448.465845367)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f9ddc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f8f6050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f91b540>, 2329448.554936854)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f8f6010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f9617d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f7254d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f8b1fd0>, 2329448.685662683)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f949dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f8b2190>, 2329449.265509908)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f725c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4113190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f8b27b0>, 2329449.018382199)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4112cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f759a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f8b3150>, 2329449.143315158)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f759f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f773350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f77a550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f794440>, 2329448.97540284)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f77ac10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b41ec2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f7a0150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f9ec9f0>, 2329449.017145424)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f852510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f794830>, 2329449.363092722)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f7a0650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f7a35d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f794ad0>, 2329449.244593733)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f7a33d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f74b290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f9ee2e0>, 2329449.009827237)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f759d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f939110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f7e6e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f8b0d70>, 2329449.198439195)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f938790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f91b3f0>, 2329449.099179337)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f7e6d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4175b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f70b590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f739590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f8ba950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f9eec10>, 2329448.973796355)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f770c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f8b1d30>, 2329448.605985352)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f70b390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b446d390>, 2329448.763079998)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4177810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f9eef90>, 2329448.752512906)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f739390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f8b2f20>, 2329448.951427301)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f8baad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f6562d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f8bb3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f679d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67c4b0>, 2329449.580210544)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f9f32d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67c9f0>, 2329449.622835475)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f679b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f68b6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67cec0>, 2329450.228157619)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f68b810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f6b3390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67d9b0>, 2329449.727968164)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f6b3190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f6bd4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f6ce210>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f9ee190>, 2329449.762054792)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f7f2590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67e270>, 2329449.803943658)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f6cdad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f6e26d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67e900>, 2329449.768100527)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f6e2250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f635450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67ec10>, 2329449.829612873)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f636f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f5078d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67edd0>, 2329450.061090802)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f507650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f7df190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f5386d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f89fb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f612550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f8b6ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f635210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f797cb0>, 2329450.814892784)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f656190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f796a50>, 2329449.3006087)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f612c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f794d00>, 2329449.314147119)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f635010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f796d60>, 2329449.331626784)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f8b6f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f919550>, 2329449.430865768)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f89f750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f91add0>, 2329449.745158022)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f7df290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f9ec670>, 2329449.853667649)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b41b72d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f516650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f558210>, 2329450.034841489)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f516590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f556ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f795080>, 2329450.212252055)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f7a3f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f596650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67ef90>, 2329450.289285508)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f56b090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f5bf110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67f540>, 2329450.401543074)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f5bef50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f406410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67f9a0>, 2329450.332305955)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f4044d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f4091d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f55b690>, 2329450.686494177)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f4084d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f5b9b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f428290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f43c750>, 2329450.711394107)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f427b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f440fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f43c9f0>, 2329450.707237176)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f611550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f522210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f4625d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67eb30>, 2329450.753215131)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f522150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f465c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f569b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f67a550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f58e310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67dda0>, 2329450.717424167)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f40ae50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f919cc0>, 2329450.021296366)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f569a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f5ad350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f43d1d0>, 2329450.662463342)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f462d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f559da0>, 2329450.22526446)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f5ad990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f43d5c0>, 2329450.687801932)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f465b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f797c40>, 2329450.052757572)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f67a0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f559320>, 2329450.213732)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f58e1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f726890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67fa10>, 2329451.640160689)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f4b6190>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f43a310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f43ef90>, 2329451.259518041)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f43ad90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f302d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f310d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f32c360>, 2329451.251757572)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f303310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f312cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f32c670>, 2329451.277747306)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f312ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f440650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f3536d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f558b40>, 2329451.005492434)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f4415d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f32d940>, 2329451.486487687)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f353350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f4ac810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f9ef310>, 2329451.114439074)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f4ada50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f3a4650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f32e6d0>, 2329451.611508415)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f3a4450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f467fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f597190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f4ccb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f55ba10>, 2329451.226542033)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f3110d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f45d710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f43d7f0>, 2329450.827445527)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f51af50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f43f1c0>, 2329450.81688139)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f4cdc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f43ecf0>, 2329450.832949983)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f596c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f32c1a0>, 2329450.912531614)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f60c7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f3a6610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f409ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f32f310>, 2329451.53036253)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f4f0a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f32f4d0>, 2329451.479940369)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f40ab90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f30bf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f2240d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f22a490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f200ec0>, 2329451.8282376)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f7b0990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f201080>, 2329451.623328073)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f21b650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f201240>, 2329451.660083602)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f22a290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f2372d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f8b3a10>, 2329451.678376832)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f237110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f262a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f43cde0>, 2329451.550581992)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f262810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f26e6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f2023c0>, 2329451.804225999)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f26e550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f330750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f3c7550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f3d9ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f31ce90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f32f7e0>, 2329451.51245781)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f31f090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f2002f0>, 2329451.676271174)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f3d9990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f32fee0>, 2329451.575615315)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f3c7c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f795a90>, 2329451.699670888)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f31cf90>\n",
      "Evaluating workflow:  24%|██▎       | 708/3000 [02:09<01:53, 20.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 714/3000 [02:09<02:02, 18.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 720/3000 [02:10<02:02, 18.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 726/3000 [02:10<01:35, 23.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 733/3000 [02:10<01:33, 24.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 736/3000 [02:10<01:41, 22.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▍       | 743/3000 [02:10<01:38, 22.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 747/3000 [02:11<01:33, 24.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 753/3000 [02:11<01:19, 28.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▌       | 759/3000 [02:11<01:55, 19.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 766/3000 [02:11<01:31, 24.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 772/3000 [02:12<01:21, 27.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 775/3000 [02:12<01:24, 26.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 781/3000 [02:12<01:45, 21.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 787/3000 [02:12<01:24, 26.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 796/3000 [02:13<01:19, 27.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 798/3000 [03:01<2:26:45,  4.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 799/3000 [03:01<2:12:02,  3.60s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 806/3000 [03:01<58:00,  1.59s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 810/3000 [03:01<38:11,  1.05s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 813/3000 [03:01<28:12,  1.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 816/3000 [03:02<21:33,  1.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 820/3000 [03:02<14:23,  2.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 822/3000 [03:02<11:23,  3.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 826/3000 [03:02<07:16,  4.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 834/3000 [03:03<03:55,  9.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 841/3000 [03:03<02:39, 13.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 848/3000 [03:03<01:42, 21.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 855/3000 [03:04<01:47, 19.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▊       | 861/3000 [03:04<01:50, 19.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 867/3000 [03:04<01:21, 26.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 871/3000 [03:04<01:27, 24.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 874/3000 [03:05<02:06, 16.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 882/3000 [03:05<01:30, 23.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 885/3000 [03:05<01:43, 20.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 891/3000 [03:05<01:20, 26.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|███       | 900/3000 [03:06<01:36, 21.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 905/3000 [03:06<01:32, 22.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 909/3000 [03:06<01:23, 24.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 916/3000 [03:06<01:36, 21.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 922/3000 [03:07<01:37, 21.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      " {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 932/3000 [03:07<01:28, 23.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 935/3000 [03:07<01:25, 24.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███▏      | 938/3000 [03:08<01:55, 17.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███▏      | 943/3000 [03:08<01:31, 22.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 951/3000 [03:08<01:29, 22.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 954/3000 [03:08<01:37, 21.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 957/3000 [03:08<01:54, 17.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 963/3000 [03:09<01:34, 21.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 972/3000 [03:09<01:22, 24.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▎      | 975/3000 [03:09<01:26, 23.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 981/3000 [03:09<01:35, 21.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 988/3000 [03:10<01:11, 28.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 991/3000 [03:10<01:25, 23.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 998/3000 [03:10<01:21, 24.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▎      | 1006/3000 [03:10<01:15, 26.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      " {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 1013/3000 [03:11<01:18, 25.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1016/3000 [03:11<01:16, 25.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 1024/3000 [03:11<01:29, 21.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1027/3000 [03:11<01:35, 20.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1033/3000 [03:12<01:10, 27.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▍      | 1042/3000 [03:12<01:40, 19.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 1048/3000 [03:12<01:15, 25.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 1053/3000 [03:13<01:37, 20.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▌      | 1062/3000 [03:13<01:21, 23.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 1069/3000 [04:01<1:19:11,  2.46s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1073/3000 [04:01<56:22,  1.76s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 1080/3000 [04:02<30:45,  1.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1083/3000 [04:02<23:37,  1.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1086/3000 [04:02<18:10,  1.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▋      | 1090/3000 [04:03<12:04,  2.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▋      | 1094/3000 [04:03<07:55,  4.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 1103/3000 [04:03<03:09, 10.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1106/3000 [04:03<02:59, 10.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1111/3000 [04:04<02:05, 15.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1114/3000 [04:04<02:08, 14.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 1121/3000 [04:04<01:37, 19.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1127/3000 [04:04<01:32, 20.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1134/3000 [04:05<01:19, 23.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1137/3000 [04:05<01:21, 22.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1143/3000 [04:05<01:21, 22.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1150/3000 [04:05<01:15, 24.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1154/3000 [04:05<01:07, 27.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▊      | 1160/3000 [04:06<01:20, 22.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 1166/3000 [04:06<01:31, 20.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1171/3000 [04:06<01:11, 25.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1174/3000 [04:06<01:23, 21.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 1182/3000 [04:07<01:26, 20.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 1186/3000 [04:07<01:18, 23.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|███▉      | 1196/3000 [04:07<01:22, 21.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|████      | 1203/3000 [04:08<01:18, 22.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1211/3000 [04:08<00:55, 32.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1215/3000 [04:08<01:08, 25.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1219/3000 [04:08<01:31, 19.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1223/3000 [04:08<01:18, 22.63it/s]Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f27bed0>\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1228/3000 [04:09<02:00, 14.68it/s]Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f202cf0>, 2329451.995213495)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f4b50d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f26b490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f200520>, 2329452.094316833)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f283650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f3c1610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f2e5e10>, 2329452.412212377)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f3c1190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f101610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f2e6510>, 2329452.393518458)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f284a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f19b6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f263250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f2037e0>, 2329452.532688534)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f262dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f12ea90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f1d5010>, 2329452.516829288)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f12f890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f286750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67e190>, 2329452.471702634)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f198690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f2e5780>, 2329452.248728408)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f2866d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f2f1550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f1d5c50>, 2329502.26041176)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f029490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f01f5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f009fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f0a0520>, 2329502.038704279)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f01f4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f0a06e0>, 2329502.149799078)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f00a2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f1b6010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f2e5fd0>, 2329501.253471008)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f0b8d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ef136d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f200de0>, 2329501.236543224)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ef11bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f070b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f164b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f0d4350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f04bf50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f0ed4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f074b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f0a3540>, 2329501.464205093)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f070e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f1d4ec0>, 2329500.763465657)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f3936d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f0a1320>, 2329502.15105899)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f0d46d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f1d65f0>, 2329502.343138993)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f04a810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f0a1be0>, 2329501.287064478)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f0eeb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f0a2040>, 2329501.194353252)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f074890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095efbf110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ef95e10>, 2329501.27848787)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095efcae50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f08cc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ef86f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ef9df10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ee23850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ee40790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f32d630>, 2329501.421720889)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ef9e590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ef946e0>, 2329502.436207602)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f08c450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ef949f0>, 2329501.184310635)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ef86e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f1d5f60>, 2329502.581502691)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f0fae50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ee115c0>, 2329502.498823406)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ee40590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095eed5590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f0a2580>, 2329502.181656503)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095eed4f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ed5c950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ee14610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095efac3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ef9ff90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095eea9550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ed07610>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ed0dc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ee129e0>, 2329502.245996853)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ed07410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ee12c80>, 2329502.442731335)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ef9ee10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f1d6740>, 2329503.085717894)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f10d350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f43c7c0>, 2329503.098314595)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ed48b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f202ac0>, 2329501.98248745)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f19ded0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ee130e0>, 2329502.12900399)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095eea9850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ee12cf0>, 2329502.186579575)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095eea9f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f070a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ed7a750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f1d4bb0>, 2329501.963204179)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f070a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f0a25f0>, 2329503.730919453)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ed7a610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ede8090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f0a3ee0>, 2329503.100013092)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ede9ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ec01190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ec0c8a0>, 2329503.113263367)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ec00dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ed6d210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095efad210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ef974d0>, 2329503.094897783)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095efad010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ed52f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f203150>, 2329503.049609415)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ec16dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ec386d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ec3c950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ec654d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ec7ad10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ec0d160>, 2329503.162097982)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ec3d050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ec0c670>, 2329502.878259601)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ed531d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f0a3a10>, 2329502.893832776)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ec38d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ec0d470>, 2329503.172483882)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ec64650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ec0c3d0>, 2329504.181801585)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ec7b3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ebb0210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eb1f000>, 2329503.886987323)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ebb06d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ecd1390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095eb49bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095eb64810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095eb7ecd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095eb8aa50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eb1cb40>, 2329503.616237116)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ecd0590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ec0eba0>, 2329503.735872283)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095eb3b6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eb1da20>, 2329503.793173121)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095eb64bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ed3d9b0>, 2329504.119564715)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095eb8a890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ef95b70>, 2329503.8310273)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095eb7eb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095eb654d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eb1d630>, 2329504.511165202)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095eaa08d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e90b150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eae15c0>, 2329505.073812571)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e90b410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ea95650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eae1b00>, 2329504.702121712)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ea95cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e96f350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ea7d4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095eb19810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ebf6cf0>, 2329504.346440572)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095eed00d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ebf6b30>, 2329504.416301985)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ea7d490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eae3230>, 2329504.800715243)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e96f310>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e9a5790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e9ba510>, 2329505.276949413)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095eb89650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e98c590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e9b83d0>, 2329505.421954568)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e98c390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e997290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e9bdf10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095eabb050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eae3d20>, 2329505.089480091)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e9970d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f0a35b0>, 2329505.114118368)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e9bdd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e9b8980>, 2329505.028235671)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e943ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ebdc490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e85c150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e8add30>, 2329505.687010223)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e857fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e8adef0>, 2329505.647052911)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e85c110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e84f590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e9b8210>, 2329505.85698081)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e84f610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e746150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eae2f90>, 2329505.898559032)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e8bc090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e75b150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e789c50>, 2329506.029953595)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e75ba10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ee75750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e8ad240>, 2329505.651730539)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e984910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e661cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e9bb070>, 2329506.407871385)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e661b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e7f9490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e684910>, 2329506.496151135)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e7f8c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e655010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e686c80>, 2329506.789687062)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e655f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e619e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e6236d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e650c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e663150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e78ac80>, 2329506.273107273)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e619c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ebf6270>, 2329506.301724445)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e650b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e78af90>, 2329506.615878362)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e6231d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e6842f0>, 2329506.517204415)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e7e2550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e57a310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e448210>, 2329507.643018921)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e57b290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e677810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e58a290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e61b2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e5ac710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e686eb0>, 2329507.200553502)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e677d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e559a20>, 2329507.350152041)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e5acd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e8ad5c0>, 2329507.02530698)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e58a990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e5595c0>, 2329507.205505345)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e656c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e4771d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e44b150>, 2329508.155077071)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e476790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e4bc6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e58b310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e499690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e5d23d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e78ab30>, 2329507.665390976)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e5d2450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e449a20>, 2329507.692281363)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e58b010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e32e820>, 2329508.762565992)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e4bca10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e44a0b0>, 2329507.460305201)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e499490>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e32b6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e214280>, 2329508.821735724)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e328ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e25d8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e215b70>, 2329509.159887172)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e25d6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e2874d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e289050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e214c90>, 2329509.147314612)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e39dd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e2b69d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e2cdd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e217af0>, 2329509.190821642)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e2b6950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e217cb0>, 2329509.144600214)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e2cdb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e2fb6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e3c58d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e3db5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e204190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e205e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e21b7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e24c050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e4497f0>, 2329509.382021743)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e287390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e55b2a0>, 2329508.310029246)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e3dbb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e32f4d0>, 2329508.774323517)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e4843d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ebf7310>, 2329508.770823079)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e204690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e215470>, 2329508.776786816)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e24c4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e32d710>, 2329509.456387689)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e2fb110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e2147c0>, 2329508.775787842)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e205d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e2149f0>, 2329508.768031914)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e21b5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e278c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f2b5f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f4f3750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f114d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f149390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f164210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f3e1990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f1824d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f189f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f1ced50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f004c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f1b5910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f007490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ef2c410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ef36f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f066390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ef6b850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ee78d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095eff6650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095eebf350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095eed3650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ef7c0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ef52b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ee06c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ed9d690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095edbb8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ed9f2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095eddb6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ec09590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095eb02050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095eb18550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ecb6c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095eb2d210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ed9ed90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ee7b510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ebca410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ebd2910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ebdcd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ea1f1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095eb022d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095edd0d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ea56290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095eba8e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095eaba950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ead2110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095eaf90d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e909190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e926cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e9484d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e96d250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ea25110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e9872d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e809290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e960ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e828450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e86d010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e9b5c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e88e290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e8a9a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e809c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e8bd710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e8e5690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e7167d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e774e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e7a5110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e77d4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e7c0390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e729110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e7374d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e9bf610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e677350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e6d3150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e730190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e5124d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e69ad50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e78e110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e51ef50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e53e690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e561810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e56a050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e5b8b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e5d0550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e5ecd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e401290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e427e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e4403d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e453d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e5ad550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e4da4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e6a3e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e301f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e6554d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e4a8490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e366cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e3815d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e349710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e84bb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e39da10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e3b7d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e26a990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e32c8a0>, 2329508.769305401)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e39d810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e449400>, 2329508.767029451)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e3b7bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e32fd20>, 2329509.878136172)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e333d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f43e890>, 2329451.9331273)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f2b5dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f202120>, 2329452.064409249)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f4f0a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f202270>, 2329452.361147562)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f1149d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f2e5f60>, 2329452.379956304)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f149190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f2e67b0>, 2329452.347101256)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f164590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f2e7150>, 2329453.383130614)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f3e1f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f200910>, 2329452.456595577)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f181e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67d4e0>, 2329452.479809078)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f18a690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f1d4520>, 2329452.524215222)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f1cdc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f1d52b0>, 2329452.549630652)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f1b5c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f43f620>, 2329452.422650038)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f1b5fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f1d5a90>, 2329500.693900336)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f0076d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f0a3070>, 2329501.40662524)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ef2ca50>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f0a3310>, 2329501.342629009)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f397750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f1d75b0>, 2329500.901396222)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f066190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f201d30>, 2329502.216411169)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ef6bf10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ee12820>, 2329501.687563501)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ee78b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ef97850>, 2329502.043812657)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095eff6bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ee126d0>, 2329502.042651211)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095eebf210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f2e7bd0>, 2329502.18047827)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095eed3e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ed3c1a0>, 2329502.478781073)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f2ced50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ed3e6d0>, 2329502.951823852)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ed5edd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ed3eba0>, 2329502.773365202)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ee82810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ed3ed60>, 2329502.849459483)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ed9dc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f1d6ba0>, 2329502.960822716)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095edbb650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ed3ef20>, 2329502.935086937)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095efef310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ec0d5c0>, 2329503.369454426)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095eddb210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ef95320>, 2329503.362480598)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ec09e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ef952b0>, 2329503.724627472)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095eb01f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ec0d9b0>, 2329503.52171656)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095eb18c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ed3c590>, 2329503.578551784)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ecb6850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ec0ef20>, 2329503.618206375)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ec3d790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eb1d860>, 2329503.905186079)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ed9ed10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ec0f9a0>, 2329503.951946285)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ede82d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eb1f310>, 2329503.929275008)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ebcabd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eb1fcb0>, 2329504.956957292)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ebd2f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eb1fee0>, 2329504.043801371)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ebdcc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eb1c750>, 2329504.242140471)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ea1efd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ebf4f30>, 2329504.167489657)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095eb02090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eb1f1c0>, 2329503.913255112)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ed6dc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eb1d1d0>, 2329504.311546653)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ea56190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ebf7a80>, 2329504.642935893)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ebabc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ebf7e00>, 2329504.706229573)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095eaba810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eae01a0>, 2329504.51250077)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ead1fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f0a3700>, 2329504.855101079)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095eaf9790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eae1400>, 2329504.720164363)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e909910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ebf71c0>, 2329504.758176297)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e924490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eae2660>, 2329505.075154212)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e948810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eae2d60>, 2329504.958095891)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e96d910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ebf7c40>, 2329504.546442724)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ea26c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eae33f0>, 2329504.860907867)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e987190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e9b97f0>, 2329505.216571186)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e80ae90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e9b9da0>, 2329505.309635607)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e961750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e9b9f60>, 2329505.327243686)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e828790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eb1e900>, 2329505.552351304)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e86ce10>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ebf6a50>, 2329505.35694608)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095eabae50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e9bbd90>, 2329505.481404521)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e88de10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ebf41a0>, 2329505.461025107)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e8398d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e9b9550>, 2329505.431443292)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095eaa1cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e8acad0>, 2329505.657550994)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e8bdc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e8ad400>, 2329505.758726555)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e8e5c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eae03d0>, 2329505.922025567)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e716ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ebf6f90>, 2329505.797729607)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e774c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eae3850>, 2329506.223434863)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e7a5450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e8ae9e0>, 2329506.067381141)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095eaa2e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f1d58d0>, 2329506.200762296)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e7b8a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e9b81a0>, 2329505.83504245)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e729c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e789080>, 2329506.283751197)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e7b8290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e78a430>, 2329506.308748066)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e9becd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e6844b0>, 2329506.620649968)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e677150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e8ae660>, 2329506.638423723)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e6d2fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e9b8050>, 2329507.370754771)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e7313d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e78a6d0>, 2329506.769856913)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e511e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e6855c0>, 2329507.06160133)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e6a29d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e8af380>, 2329506.688036294)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e6ca190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e78a900>, 2329506.764581706)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e51f610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e78b700>, 2329506.734716924)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e53dc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e8ac050>, 2329507.082698336)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e561610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e558ad0>, 2329507.073209276)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e569f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e559c50>, 2329507.687344577)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e5b9150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e78be00>, 2329507.108024263)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e5d0c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e55b230>, 2329507.36379708)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e5ecb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e55b850>, 2329507.479051021)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e401150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e448050>, 2329507.736697846)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e427d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e684de0>, 2329507.517150834)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e4409d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e448de0>, 2329507.67185972)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e453c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e9b9cc0>, 2329507.558178965)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e5ad150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e44b4d0>, 2329507.847029747)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e4da150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e78b930>, 2329507.910960185)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e729710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e44b770>, 2329508.2581702)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e302690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e44bb60>, 2329507.956751453)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e655c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e32cde0>, 2329508.168387883)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e4a80d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e55b5b0>, 2329508.085239732)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e486390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e32dfd0>, 2329509.138816819)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e381c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e559be0>, 2329509.0196422)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e46e890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e32dcc0>, 2329508.282854168)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e4d5f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e44aba0>, 2329508.867450007)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e26a790>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f2da4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ea6b010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e606b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f43cd00>, 2329452.249922336)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f2dac10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eb1def0>, 2329504.442435346)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ea6aed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eb1fa80>, 2329506.23506103)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e606950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e06d110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e6858d0>, 2329510.110715814)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e06d650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e11d4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ee10980>, 2329510.19247328)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e3b45d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e53d250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f0a1160>, 2329510.363585561)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e53ee50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e086690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ea30d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095eb61c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e2e9470>, 2329510.480029201)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095eaa0950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e2ea900>, 2329510.619110454)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095eb60590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e116210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095eb0e190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f9efc40>, 2329510.629010778)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e1e57d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f9eedd0>, 2329510.549874662)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095eb0f310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ed89f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e2ea5f0>, 2329510.781507532)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ed89410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ed0dd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ee62ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f9ed1d0>, 2329510.999495461)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e206090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f9ec1a0>, 2329510.681732883)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ee60cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e3dbad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ef72290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e1986d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e1a1510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e1b0f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e1dd190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e1e46d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e1e6310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e1f4190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e1f5e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e00d4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e034390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e04cdd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e0610d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e10fb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e0654d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e067190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e205510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e1984d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f1d4360>, 2329510.316146348)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e604210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e2166d0>, 2329510.18552249)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e1a1390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e685be0>, 2329510.217733331)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e203c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ebf6ac0>, 2329510.142642808)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e075ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e2178c0>, 2329510.017813654)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e04c790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f9ee900>, 2329510.551907732)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e3dba90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e2e9a90>, 2329510.656640347)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ef73950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b639c590>, 2329509.696428245)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e199090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b720aa50>, 2329509.526173203)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e1a1a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b720b930>, 2329509.467721001)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e176610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4e8b930>, 2329509.737302491)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e1c13d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b71cf310>, 2329509.581843099)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e1c3d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e2e8bb0>, 2329509.862462455)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e1e6a50>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7109a90>, 2329509.719346944)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e2cc810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453e890>, 2329509.768772588)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e1f6190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a150346e0>, 2329509.928951144)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e115010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7209160>, 2329509.858209302)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e024110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e558ec0>, 2329509.89114639)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e10c2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ec0e6d0>, 2329509.869077346)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e067490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e2ebb60>, 2329510.214241485)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e19be50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e477650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f0a2f90>, 2329510.335790082)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e4764d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ef1b750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e215e10>, 2329510.977413543)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ef1a110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e258f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f1e5890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f91b380>, 2329510.976131971)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e24d810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f918050>, 2329511.004880974)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e554610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f183290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f91a7b0>, 2329510.954505252)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f182990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f26cd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f2e5c50>, 2329511.475967882)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f26c310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ee60e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f134510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ed3fa80>, 2329511.120507406)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ee60c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f219710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f301c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f9189f0>, 2329511.595913986)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f218f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b446edd0>, 2329511.441102071)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f302650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f02a950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b446f0e0>, 2329511.280820508)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e4759d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f084050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f4ceb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095efdff10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ee0ce10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f0ef510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f032510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f91b2a0>, 2329511.091714756)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f136790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f9ec280>, 2329510.810150642)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095eff7610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f9ef070>, 2329510.843743413)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ee0c610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f918f30>, 2329510.805369504)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f0f0310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f1d7ee0>, 2329510.929621108)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e9dad90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f9eda90>, 2329511.390293906)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f0d4550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b446cad0>, 2329511.602675298)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f4cec50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f3c4190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b446e820>, 2329511.802232641)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f3c7fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f422a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f919940>, 2329512.050139049)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f422b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f3a7110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b446d160>, 2329511.560083745)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f3a7a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f348310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b72b2040>, 2329511.9613904)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e8f1b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f3f0bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f495150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f5f58d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e2eb230>, 2329511.872684841)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f3f0850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b72b1b00>, 2329511.750108741)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f495f10>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b72b1160>, 2329511.815147739)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e1ddd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f310690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f91a660>, 2329511.806328383)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f312290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f369bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f6bce50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f91aba0>, 2329511.966941144)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f031b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b746d320>, 2329511.846422678)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ee43e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f384ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b746dbe0>, 2329512.104768153)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f385790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f487310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ef1bcd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f21af10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f45e250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f40ab50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b446c9f0>, 2329511.675494734)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f465c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b446f380>, 2329511.66557574)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f45e090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5eda350>, 2329512.088212284)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f486910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b446f460>, 2329511.666701002)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095efc1ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b446dd30>, 2329511.776580582)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f40bbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f613110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5edad60>, 2329512.237744061)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f3c71d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f63f8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b446de80>, 2329512.603942171)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f63f710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f332c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f7f1150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6774ec0>, 2329512.367448581)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f6e1a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6776f90>, 2329512.304282917)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f7f1550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f891a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f520d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f8f59d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee7f0b40>, 2329512.46543926)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e7f2310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee7f2900>, 2329512.48144129)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f521010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee7f11d0>, 2329512.447834445)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f8f46d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f8b6b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b446d940>, 2329512.499972312)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f8b5c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f6a4490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f7e7190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f9ec2f0>, 2329512.700098947)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f5c6550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f919860>, 2329512.646495951)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f7e6250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f3c6b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f719910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b59132a0>, 2329512.483393792)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f58fe50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f9ef540>, 2329512.493339576)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f719b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f8b8dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b72b06e0>, 2329512.621687845)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f8ba790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e072fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5910130>, 2329512.693201667)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f625c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f3c70d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f385f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f63aed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f91b8c0>, 2329512.275899705)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f63bc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5913070>, 2329512.888515778)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f3f0c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5fb99b0>, 2329512.722013026)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f80eb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f9f3090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5fb8600>, 2329512.760171236)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f9f3ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f974e90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b744e970>, 2329513.06682604)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f974950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f732b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee7f0520>, 2329560.827671845)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f732290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f96ee10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b744c750>, 2329561.16728294)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f867710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f96edd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f91a510>, 2329561.049370652)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f787c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f4cea10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b41b6990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f918bb0>, 2329561.271934955)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f4cf090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f91a0b0>, 2329561.211685239)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b41b6890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b41dce10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b68ddcc0>, 2329561.304665646)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f3b4d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f56b510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b413bdd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b68dec80>, 2329561.675679084)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f56a4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b67f2200>, 2329561.334788835)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b41394d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f983410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b67f3150>, 2329561.339165491)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f9825d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4174710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4147b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f857810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f9caa90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5fbba10>, 2329512.706970357)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f9c9710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b746fe70>, 2329512.777706355)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f855d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b67f08a0>, 2329561.492695131)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f576490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b744f1c0>, 2329561.484993899)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4147a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f993fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee7f1470>, 2329561.593455573)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e023250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4a5e2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b71cd240>, 2329562.668956403)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b47d2e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b484df50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b41f5450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4403310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b442d190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4602510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b44c71d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f93fdd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b461ab10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4d21cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7191630>, 2329561.722453705)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b442cc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b744f000>, 2329562.775509137)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4602850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7192d60>, 2329561.560686411)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b44c6950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5913cb0>, 2329562.603383745)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b461b790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453d4e0>, 2329562.838995566)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b484fd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b67f0e50>, 2329561.465670626)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b41f5710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7190440>, 2329561.647025007)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4400150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee744fa0>, 2329561.935039274)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f93fd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f918d70>, 2329561.911123853)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4d20510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4a22090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee68c7c0>, 2329563.135747382)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4a20e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6998d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453c590>, 2329562.63109407)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6998d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f999fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453da90>, 2329562.47418385)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b47da8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4805a90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453d7f0>, 2329562.52823624)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b44db6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5440fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b68dc4b0>, 2329562.644872849)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5440d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4a60590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453def0>, 2329563.041846521)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4a62590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5442f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b48d7010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b418c690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f77bf10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b496ec10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4d79190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b60df950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5994450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b599e210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee7478c0>, 2329562.325271319)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b599e010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453eeb0>, 2329562.766969885)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b54ae310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b67f30e0>, 2329562.091890471)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b48d61d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee68f000>, 2329562.989050858)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f5f6810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b71ce4a0>, 2329563.322074323)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4d21b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b71ce3c0>, 2329563.35457795)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b48c9c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453f540>, 2329563.397152614)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4d78090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b744e3c0>, 2329562.324080534)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b60dda10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453dbe0>, 2329562.372948383)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5995250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b60ee290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453ed60>, 2329563.450879077)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6095c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee78b5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4f95d30>, 2329563.148908158)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b47fe410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4512250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b63fe450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4f94c20>, 2329563.361671885)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4512f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4f97000>, 2329563.57270715)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b63ff2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b48457d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4f96e40>, 2329563.50689537)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b51c62d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b54c8090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4504490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4951e10>, 2329563.45744839)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4506a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b56d5890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b49526d0>, 2329563.655254998)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4507390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6935550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b644bbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4952120>, 2329563.525966352)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6935bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b71cd160>, 2329563.474886937)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6449a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b72c4350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4f944b0>, 2329563.513336376)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b72c4d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6f1f910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4951be0>, 2329563.555469466)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6f1e490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b51c4510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b53c1750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee7cc650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6019a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b71cc8a0>, 2329563.470702573)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b54c82d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6934710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4f97540>, 2329563.058052732)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6936750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4f94f30>, 2329563.083621001)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b60c5c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b68de430>, 2329563.763232905)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b51c6050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee745470>, 2329563.853506968)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b53c3a50>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5fbb540>, 2329562.791684951)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee7ccb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee668290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4952b30>, 2329563.958097618)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b732e950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a154cb750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b756c750>, 2329563.83147632)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a154cbe50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a154c5710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4f97d20>, 2329564.113646921)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a154c7050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4783e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b7099310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4f95470>, 2329564.29777654)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b445ff90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6f99d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b49512b0>, 2329564.181821804)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6f99dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a1619bb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b756c590>, 2329564.325745189)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee558250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a1536c490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5a86890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4f958d0>, 2329564.352550671)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee9cd0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4845650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4950ad0>, 2329564.390332403)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4845890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a154c8b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b489b890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a159bb7e0>, 2329564.323676261)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a154c9ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a159bbbd0>, 2329564.235505718)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b515a150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5c2d890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5e25470>, 2329564.419221657)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6eecc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5efe890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5833990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4953620>, 2329564.108298679)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b67b2990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b67f3000>, 2329564.195626001)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5e95750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b49535b0>, 2329563.768038306)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5efff10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4952040>, 2329564.385590679)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5c2c3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b756c7c0>, 2329563.836348351)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5830d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee98c950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5df7770>, 2329564.848809327)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee98cfd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b69b35d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a150b0fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5e25550>, 2329565.204307575)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b69b25d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f43f540>, 2329565.027005297)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b62eee50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b7443b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5e27e70>, 2329564.709389426)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b7442990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b569fcd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4a75cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f43d4e0>, 2329564.970121895)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a14fee610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f43e3c0>, 2329565.094892527)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4a74c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee92e850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f43dd30>, 2329565.316883887)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee92c910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5d27e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b455c0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f43d2b0>, 2329565.143960356)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4a7a5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f43c910>, 2329565.43109977)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b455f4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b45df910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a153750d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b418d150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b49cf0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5e25ef0>, 2329564.45134347)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6e38c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b49538c0>, 2329564.579288052)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a15377b90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453d6a0>, 2329564.609559276)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b49cee50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5f2e590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b756de80>, 2329565.132764575)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee98f250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b49522e0>, 2329564.561210216)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5f2d450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee606990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4832e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f32fbd0>, 2329565.063299808)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a15d9c390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f32c050>, 2329565.542808247)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4830210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b59c1e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a159bbaf0>, 2329565.418987446)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b59c18d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a16118910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a1505e010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f203b60>, 2329565.411137228)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b66d2450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f200b40>, 2329565.439859426)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a1505d250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b45e9010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f43cf30>, 2329565.4775125)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b70a79d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6e9b5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b756d860>, 2329565.679900957)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6e98f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5270ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b58f20d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f43f1c0>, 2329565.621231065)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b45e75d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee613cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f558520>, 2329565.968960692)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee611c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee633790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4e4fd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b483e1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b457fc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f43c980>, 2329565.113137854)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b457eed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f32e9e0>, 2329565.17043944)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b483d090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f32f310>, 2329565.349427663)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4ee6590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b49533f0>, 2329566.175707323)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b58f05d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5ee2050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f32cc20>, 2329565.408445618)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5ee30d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5e24f30>, 2329566.179406904)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee630490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5c49c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5e00b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee71b850>, 2329566.01777029)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4ea3f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67e7b0>, 2329566.031074128)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5e03c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a156375d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f55b000>, 2329565.89640856)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b572ea10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b64cea90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67fd20>, 2329566.087135745)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5f0af10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6234e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5e272a0>, 2329565.987959725)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6235750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a154ed810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67f850>, 2329566.159703711)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a154ec5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee68b490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a1542f450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67db70>, 2329566.619608452)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee689c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67d080>, 2329566.421623652)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4a84a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b65227d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b754e450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67ecf0>, 2329566.083172762)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a1542da90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67e900>, 2329566.327599646)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b754c650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6b21f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b676afd0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67e350>, 2329566.288624994)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b626eb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67fa80>, 2329566.633561304)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b67691d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4839050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f8b04b0>, 2329566.349006174)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b483ac10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a161dab10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f8b35b0>, 2329566.476383248)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a161d8310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4978c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b572cc50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a1549e450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f8b1a20>, 2329566.401760083)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4979490>\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f32f5b0>, 2329566.28388321)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b572cf90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67fd90>, 2329565.980272112)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a1549dad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b651fc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f8b29e0>, 2329566.693888289)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b651cc50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b610c210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5d7a710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f8b0980>, 2329566.793030266)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a1549ea90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f8b1710>, 2329567.099242139)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5d7b1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5df06d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b720a660>, 2329569.087599131)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b488dc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b610d4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f8b2740>, 2329566.888089089)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b610dbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b535f010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b7159d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f796f20>, 2329566.787759008)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee98fd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f796660>, 2329567.029845988)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b715bd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b47891d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6ecb490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f7946e0>, 2329567.010239835)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4789690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f795630>, 2329567.104610079)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4915d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6ec9210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f794210>, 2329566.971595287)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6ec81d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b722b690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f7951d0>, 2329567.140714716)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b722a310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6b22690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b75e9b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67fcb0>, 2329567.164460393)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b75d2910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f32f460>, 2329567.085278001)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b75ea190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee803b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b45dfa50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f795400>, 2329567.092725604)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a153d7b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f8b3850>, 2329566.692510532)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b45df550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6337a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f794e50>, 2329567.091567811)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6609050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5182010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b720be70>, 2329567.46032675)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5183450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b61ca410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b49b6a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b72087c0>, 2329567.461792915)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee90a190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b720ac80>, 2329567.574222607)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b49b56d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6f6a490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f795320>, 2329567.711015607)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6f6a550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee803550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67f8c0>, 2329567.508316683)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee8026d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6e5e450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b51341d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67de80>, 2329567.958818245)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a15390c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6b595c0>, 2329567.868083622)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5135490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b70ef8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b476e750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6b5aeb0>, 2329567.936142542)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b70ee350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b41517f0>, 2329567.926936857)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b476e110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b57ce950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4150c90>, 2329568.07909829)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b475e5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a1553f1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b7490e90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f795550>, 2329567.297427793)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a1553d6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee7e8e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f794d00>, 2329567.438719168)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b7492f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b720b620>, 2329567.381727309)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee7e8250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6be1050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a153e2950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4153540>, 2329567.998014697)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b53e5150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f8b0210>, 2329568.167746116)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a153e1d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b70688d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f2002f0>, 2329568.005276951)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5c39710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b49631d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7209010>, 2329568.080607808)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5e99910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b52a7010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b68d71d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a15521d30>, 2329568.467876632)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b52a6710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b41519b0>, 2329568.324574341)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b68d4e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5f6c910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f795da0>, 2329568.500771027)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5f6e710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b64f8910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a15522cf0>, 2329568.742314003)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b64fa190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6357e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a15523850>, 2329568.402153692)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b63550d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4824b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6b254d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b649e210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee8a3210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b52bde10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4153a80>, 2329568.024057657)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b649e110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b588d950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f796cf0>, 2329567.906709295)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4a6d390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a15523b60>, 2329568.767942424)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b631b910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4152510>, 2329567.807949548)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b57eb510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4151cc0>, 2329568.152140355)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee8a3050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f558bb0>, 2329568.211638875)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b52bec90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b69736d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee92ba10>, 2329568.855165101)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6970690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6440990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b7097290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee92ad60>, 2329568.852695154)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6441190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7208a60>, 2329568.912585683)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b7094050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6813d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee928d00>, 2329568.978680987)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6811b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6811350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee92b770>, 2329569.421618474)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6811150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b486f190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6fbd410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee92aeb0>, 2329569.077952001)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b75649d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f8b0670>, 2329568.989406096)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6fbd490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a153e0e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee92bd20>, 2329569.139224376)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee6b5f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b730bb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f797f50>, 2329569.006249964)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5b32d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b75656d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee92baf0>, 2329569.276674509)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5dab6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4ae9c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee8f0a50>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b72b55d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b532af10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6edf510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6589d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a15520c20>, 2329568.617325458)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee8f0310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b49520b0>, 2329568.69930203)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b532a4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a155203d0>, 2329568.983568898)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b72b7550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6278910>, 2329569.315595887)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4aeb3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a15520ec0>, 2329568.847799586)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6edd550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4150b40>, 2329568.970765054)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee6b3d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a15443790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7209be0>, 2329569.245783452)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b698a0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5e5fb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6278750>, 2329569.530528065)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5e5f990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4963550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b41510f0>, 2329569.424014735)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6753a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b59fa590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b720a3c0>, 2329569.39074554)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b59f91d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b71d3210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6381710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b627a0b0>, 2329569.545742972)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b691da90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e32d7f0>, 2329569.614762391)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b60e2fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b62c1910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a153f5250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e32f540>, 2329569.797817751)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b62c1f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e32c750>, 2329569.736950541)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a153f5e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b476af90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a155239a0>, 2329569.72865751)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4769710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6986690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b698b6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5c5a970>, 2329569.809911567)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b65adc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6643190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5bf8e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5c5bbd0>, 2329569.678991686)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6640590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5c58e50>, 2329569.738201264)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b7483810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5d23a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee93e1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5c595c0>, 2329570.865919953)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b69f8a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5c5b930>, 2329569.821629649)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee693c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b66841d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5c59b00>, 2329570.02081126)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b69858d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5c58b40>, 2329569.801453925)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5eae890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee7eb6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5c5bd20>, 2329570.134080438)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b66f9f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b7483090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6278d00>, 2329570.083704225)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b7480410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee7a8390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5a5e2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5c59240>, 2329570.278757338)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b67ef490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7208750>, 2329570.259450804)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4aed4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee986050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e44b930>, 2329570.54308896)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee985a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6563650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b55642f0>, 2329570.322114599)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5c4d550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b7555290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b627b0e0>, 2329570.305195156)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b7556bd0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5e34f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5564d70>, 2329570.371174743)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5e36190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6e977d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b57edb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5c58910>, 2329570.377658292)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6e94bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b63c40c0>, 2329570.609825447)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6303e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5170510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b7135f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4955290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a160a1590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4151a90>, 2329569.883300526)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4956950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee92b620>, 2329570.052007391)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a160a2a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b63c6040>, 2329570.541877039)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5172e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b63c5240>, 2329570.6313967)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b7134cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4748ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b63c74d0>, 2329570.685658653)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b474b410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b486bb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b41ef7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b63c7150>, 2329570.709468261)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b486b750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e32cbb0>, 2329570.789013549)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b41ede50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a150a0a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b63c5c50>, 2329570.790173315)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a150a3e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e024cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee92bb60>, 2329570.833717032)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee930dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b66e0990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee929ef0>, 2329571.061239982)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b66e2c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d909c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d92a4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee92b930>, 2329570.970216829)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d909850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4ef1b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d952b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5d65da0>, 2329571.078548881)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d953250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f5d5f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5334d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5dc51d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b63c6820>, 2329570.565322911)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5cbec10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5c58c90>, 2329571.063186509)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d929c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b63c6c10>, 2329570.611031743)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5dc6590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4ef3650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b63c6350>, 2329570.617219551)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4ef0ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b63c6ac0>, 2329571.351485785)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d950ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5d65be0>, 2329571.105914756)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f5d6b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b54ba190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b55667b0>, 2329570.697099581)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b54b9690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e02bbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e1c2cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d99f3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4675400>, 2329571.46279946)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e0297d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4676cf0>, 2329571.457367185)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e1c3010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4674130>, 2329572.401926051)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d99f290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d9c2590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d9217d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d9d3790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b46758d0>, 2329571.412292159)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d9c2850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b46774d0>, 2329571.423880456)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d9c1ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4677690>, 2329571.709406492)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d9d3590>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b713fcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b46772a0>, 2329571.492816012)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5dbb510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d978610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b46742f0>, 2329571.598596215)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d97b9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d92a090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d979850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d85e8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4fd2cf0>, 2329571.541556975)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d979410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4fd0fa0>, 2329571.719689177)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d85ef90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d8773d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5c5b8c0>, 2329571.747045775)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d877290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d88be10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d8a3ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e44a7b0>, 2329571.71408022)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d88bcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4677a80>, 2329571.70440198)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d8a38d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d8a5c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d9e0d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d8104d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4fd2f20>, 2329572.603276907)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d850750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4fd2580>, 2329571.785392345)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d8a5a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b46767b0>, 2329572.462342375)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d9e14d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4675940>, 2329572.413258885)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d810bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e02b010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4fd0bb0>, 2329572.187872767)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e029d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d731ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee929390>, 2329572.566202)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d7329d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d8ea4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d755e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b583f5b0>, 2329572.330748769)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d8e9ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4e8b3f0>, 2329572.931154975)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d755c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee7c8150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d771e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5d64980>, 2329572.277100336)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee7c8b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4e88750>, 2329572.551457835)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d771c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d791690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4e8a820>, 2329572.541514042)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d791550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d7a13d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4e8a200>, 2329572.654188688)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d7a1290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d9b1390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d7c0450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4fd1630>, 2329572.540285074)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d7049d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4fd2d60>, 2329572.432618107)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6358cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d8c47d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d8e0e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4fd3310>, 2329571.796252724)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d8e0d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4fd0750>, 2329571.820513124)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b56ac810>\n",
      "Evaluating workflow:  41%|████      | 1232/3000 [04:09<02:09, 13.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████▏     | 1243/3000 [04:10<01:42, 17.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1249/3000 [04:10<01:35, 18.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1254/3000 [04:10<01:19, 22.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1260/3000 [04:11<01:34, 18.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1266/3000 [04:11<01:24, 20.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1270/3000 [04:11<01:14, 23.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1273/3000 [04:11<01:35, 18.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1279/3000 [04:12<01:31, 18.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1287/3000 [04:12<01:13, 23.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1290/3000 [04:12<01:25, 20.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      " {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1296/3000 [04:12<01:26, 19.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1300/3000 [04:13<01:11, 23.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1303/3000 [04:13<01:19, 21.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▎     | 1306/3000 [04:13<01:29, 18.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1314/3000 [04:13<01:21, 20.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1320/3000 [04:14<01:06, 25.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1324/3000 [04:14<01:04, 26.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1327/3000 [04:14<01:27, 19.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1335/3000 [05:01<1:09:09,  2.49s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▍     | 1341/3000 [05:02<39:45,  1.44s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▍     | 1348/3000 [05:02<19:41,  1.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1352/3000 [05:02<13:38,  2.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1355/3000 [05:03<10:42,  2.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1358/3000 [05:03<08:36,  3.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▌     | 1363/3000 [05:03<05:58,  4.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1369/3000 [05:04<03:29,  7.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1374/3000 [05:04<02:19, 11.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1380/3000 [05:04<01:57, 13.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1387/3000 [05:05<01:27, 18.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▋     | 1394/3000 [05:05<01:09, 23.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1397/3000 [05:05<01:17, 20.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1404/3000 [05:05<01:07, 23.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1412/3000 [05:05<00:58, 27.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1418/3000 [05:06<01:13, 21.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1423/3000 [05:06<01:02, 25.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1426/3000 [05:06<01:01, 25.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1433/3000 [05:06<01:10, 22.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1441/3000 [05:07<00:55, 28.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1445/3000 [05:07<00:55, 28.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1448/3000 [05:07<01:16, 20.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1455/3000 [05:07<01:07, 22.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▊     | 1462/3000 [05:08<00:54, 28.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1466/3000 [05:08<00:58, 26.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1472/3000 [05:08<01:16, 19.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1475/3000 [05:08<01:28, 17.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1483/3000 [05:09<00:59, 25.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1487/3000 [05:09<00:56, 26.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|████▉     | 1493/3000 [05:09<01:04, 23.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1496/3000 [05:09<01:15, 19.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1501/3000 [05:09<01:03, 23.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1504/3000 [05:10<01:22, 18.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1507/3000 [05:10<01:41, 14.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 1514/3000 [05:10<01:19, 18.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1520/3000 [05:11<01:10, 20.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1523/3000 [05:11<01:16, 19.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1532/3000 [05:11<00:56, 25.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████▏    | 1539/3000 [05:11<01:02, 23.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████▏    | 1542/3000 [05:12<01:20, 18.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1551/3000 [05:12<01:08, 21.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1554/3000 [05:12<01:06, 21.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1560/3000 [05:12<01:12, 19.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1566/3000 [05:13<01:06, 21.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1571/3000 [05:13<00:53, 26.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1574/3000 [05:13<01:19, 17.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1577/3000 [05:13<01:21, 17.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1580/3000 [05:14<01:17, 18.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1586/3000 [05:14<01:00, 23.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1592/3000 [05:14<01:12, 19.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1596/3000 [05:14<01:06, 21.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1599/3000 [06:01<1:36:15,  4.12s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1603/3000 [06:02<1:04:39,  2.78s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▎    | 1609/3000 [06:02<35:20,  1.52s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      " {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1614/3000 [06:02<19:53,  1.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1622/3000 [06:03<08:57,  2.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1625/3000 [06:03<07:09,  3.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1628/3000 [06:03<06:05,  3.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1633/3000 [06:04<04:13,  5.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▍    | 1638/3000 [06:04<02:44,  8.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1641/3000 [06:04<02:08, 10.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▍    | 1646/3000 [06:05<01:45, 12.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▌    | 1653/3000 [06:05<01:12, 18.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▌    | 1663/3000 [06:05<00:48, 27.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1670/3000 [06:06<01:04, 20.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1677/3000 [06:06<00:53, 24.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1685/3000 [06:06<00:49, 26.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▋    | 1688/3000 [06:06<00:56, 23.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▋    | 1694/3000 [06:07<01:06, 19.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1703/3000 [06:07<00:47, 27.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1709/3000 [06:07<00:51, 25.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1715/3000 [06:08<01:11, 18.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1723/3000 [06:08<00:42, 30.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1727/3000 [06:08<00:57, 22.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1731/3000 [06:08<01:01, 20.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1744/3000 [06:09<00:37, 33.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1749/3000 [06:09<00:58, 21.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1753/3000 [06:09<00:59, 20.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▊    | 1758/3000 [06:09<00:49, 25.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▊    | 1762/3000 [06:10<00:51, 24.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1766/3000 [06:10<00:54, 22.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1769/3000 [06:10<01:12, 16.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1778/3000 [06:10<00:54, 22.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1784/3000 [06:11<00:50, 23.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1787/3000 [06:11<01:11, 17.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1791/3000 [06:11<01:02, 19.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d6245d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b45614e0>, 2329572.985039799)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d6244d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d67f110>\n",
      "Evaluating workflow:  60%|█████▉    | 1794/3000 [06:12<01:53, 10.67it/s]Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4e8b850>, 2329621.45513545)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d67f3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d63c610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7108a60>, 2329621.476334778)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d6d9390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d8cce90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d71be90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d60a890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d613290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b45633f0>, 2329572.761830403)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d6138d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4674910>, 2329572.642993955)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d71b190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4675b70>, 2329573.197211273)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d8cc250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4fd14e0>, 2329572.507482896)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d60a650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d5036d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b710b460>, 2329573.254663714)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d66ba10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d7a2210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b70e7ee0>, 2329621.497364293)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d8a2150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d5eadd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4562430>, 2329621.824713244)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d5eacd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d5ac4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b71090f0>, 2329622.024380155)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d5ac6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d7c3050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d541e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d645690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d56c6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d5e8bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4fd3cb0>, 2329621.916026988)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d7c0090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b710aac0>, 2329573.19117206)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d542510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7109e10>, 2329621.651067829)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d644910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7109010>, 2329621.423307475)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d644710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4e88280>, 2329621.893420062)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d5e89d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d5618d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7109e80>, 2329621.672142829)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d561fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d90b9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b710a430>, 2329573.929968538)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d7aa210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d4eb650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b65b0980>, 2329621.701741657)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d4ea710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d341390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee9ab380>, 2329623.316653238)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d341610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d482c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d4aa0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d4c6790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7084d00>, 2329623.149780352)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d482a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7087460>, 2329622.13935076)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d4a9e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7109940>, 2329623.31240345)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d4c7190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d514490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4677850>, 2329622.606631334)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d3afa90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d4aba50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b70e6660>, 2329622.802782563)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d4abb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d35bf50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4fd3a10>, 2329622.668655468)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d35bd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d2a6950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b65b1780>, 2329623.258285049)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d2a6b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d4c6590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d5a2210>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d3e6490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d3f4b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d31d190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d22c2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d250710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6857070>, 2329622.784367077)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d31cb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4e89e10>, 2329622.543390052)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d914390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b70e7e00>, 2329623.748686958)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d5a0d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b65b0210>, 2329624.131283595)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d3e6b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee9a9a90>, 2329623.835414048)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d3f4950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b65b1be0>, 2329623.833086097)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d22c950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6854d70>, 2329622.901363877)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d250dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d2d7510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b70e5080>, 2329623.505943298)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d2d7410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d6d02d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b71086e0>, 2329624.079099335)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d17fb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d1c0590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7114f30>, 2329624.165013855)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d1c0490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d2bc810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b62b3cb0>, 2329624.330684236)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d3af850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d2ef9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d2fad10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d2885d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b62b29e0>, 2329623.558151296)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d368a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b62b24a0>, 2329623.775767766)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d2fab50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b62b10f0>, 2329623.48746969)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d2eff10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d0c51d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d048690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b60ba970>, 2329624.436993595)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d048490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7001b70>, 2329624.850153992)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d2bd290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d027210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b62b0d70>, 2329624.550029371)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d02b1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cf6fed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6854e50>, 2329625.165842194)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cf6e890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cf93a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee9a9400>, 2329625.495350482)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cf93b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cfa54d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d024e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b639f9a0>, 2329625.044937365)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cfa4b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b639f540>, 2329625.124876071)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d025f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ce11910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b506d0f0>, 2329625.579992941)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ce12750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d0ca110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5ea9fd0>, 2329625.98557778)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ced1110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cdab010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cea7c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cebc310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b70e6350>, 2329625.734198009)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cf9ca50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ce5a390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b76eb460>, 2329625.776658751)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ce5a290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5eaa190>, 2329626.350080824)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cdab310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b506c1a0>, 2329625.520709752)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d5309d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cd20d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a150366d0>, 2329626.420708647)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cd20bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cd84210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a150355c0>, 2329626.547318244)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cd84410>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d2f8b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cee7f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cc27510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a15034050>, 2329626.419635313)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d070210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cdee050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a15035940>, 2329626.442679022)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ceb1090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5ade5f0>, 2329626.876077419)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ccf4890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a150377e0>, 2329627.437158324)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cf44390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ccbe210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a15036eb0>, 2329627.116254977)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ccbef90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ccd5850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e0144b0>, 2329627.232495911)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ccd6ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cbb0950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e017540>, 2329627.515676609)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cbb0b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cb21150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cb3c650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cc541d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5adc980>, 2329627.055227169)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cb21010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5adc600>, 2329627.219055752)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cb2ff90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e016190>, 2329627.165710616)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cd93410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ca691d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ca93f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ccc6550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e015da0>, 2329627.831905861)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ccfa710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e0154e0>, 2329627.93793273)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ca93e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ca20350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ca54de0>, 2329627.998759491)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ca6be90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ca556a0>, 2329627.750234162)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ccc73d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cad3cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4950980>, 2329632.120098364)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c88e210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ccd4350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095c855f60>, 2329628.845279525)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c90f590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ca78c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c9af990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ca555c0>, 2329628.192634313)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ca78dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b60bb310>, 2329628.370897357)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c9aff10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c85b8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095c7540c0>, 2329629.750214213)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ca9c2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cadce90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095c754bb0>, 2329629.693686725)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c749a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c78fd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c9f7190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095c958ec0>, 2329629.707238083)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c9f7650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c765490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c74d090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095c756900>, 2329629.701963647)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cbf5410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b639ef90>, 2329629.699295173)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c74ba90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c71c510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095c959780>, 2329629.696510546)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c71c2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c76d110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095c757620>, 2329629.759428073)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c76c410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c9401d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c62cfd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c9afe90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c8f8290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c71c150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c723890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095c95a040>, 2329629.68992074)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c78fc90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095c856f90>, 2329629.342264522)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c8089d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e0162e0>, 2329629.357026398)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c723690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095c757d20>, 2329630.087208453)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c940950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095c757ee0>, 2329631.038869763)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c6236d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095c856d60>, 2329630.002114377)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c9aff50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095c8578c0>, 2329630.08305765)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ca84890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c62f010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c64ef10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d86ddd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d7e0310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d7e6590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b7135cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d62dbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d6d3090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d5012d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d516650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d6bf710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d5c2090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d5dba10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d44b8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d593090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d471910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d31c990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d3399d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d34f8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d306510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d37b750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d7a2810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d283710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d2b35d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d26eb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d2d5290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d13f8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d150610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d178d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d2ea190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d199e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d1d41d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d1f2890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d1b21d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d2b36d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d009610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d05e750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d06ad10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d08e810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d403c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d1b2d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d3e4e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d1e2d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d055f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cf18e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cf34210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d07d350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cf53390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d06ba90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cfd0f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cfd2b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d1e3390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ce3bb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ce73510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cf90c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ce03d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ce38bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ce63510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ce8bd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cee2710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cef2f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cd060d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cd2f250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cd8a750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cebc750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cea7510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d37a490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cdc5990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cc11010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cc4ead0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cd93210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d1e3350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cc7da10>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cdd74d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ccada10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ccc6250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ce38f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ccf8dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cb57e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cb9f510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cbb3310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cbd8550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ce7cc50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ca06d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cb8d7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cc96e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ca41290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095caaa350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095caca390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cad2c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cadc4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095caf8250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cb2a610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c91eb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c925c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c937290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ce79290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cab7810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c9f6690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c821310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c961490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cadcd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cba8650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c88c310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ca38910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c8b4b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c993710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c8424d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c885350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c8d8c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095c855240>, 2329629.344710985)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c885a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095c856660>, 2329629.354366941)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c8d9310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095c630910>, 2329630.183885206)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c74d550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f1d6eb0>, 2329630.31030013)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c64edd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4e8bee0>, 2329572.407190982)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d729d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4560360>, 2329572.501636472)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d7e0810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4561e80>, 2329573.073941471)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d7e6c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4561400>, 2329573.07997953)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d6102d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7109390>, 2329573.202798051)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d62f1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b710ad60>, 2329573.157173612)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d6d2e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4e8a740>, 2329573.306790802)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d501190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b710a740>, 2329573.285005694)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d516a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b710a970>, 2329573.37239471)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d6bf510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b710acf0>, 2329621.736068432)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d5c27d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b45604b0>, 2329621.751509541)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d5dbf10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a15520fa0>, 2329621.939623865)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d44b550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b70e7b60>, 2329621.794391005)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d592f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7084b40>, 2329623.476313147)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d471750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b45611d0>, 2329622.233557688)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d31d050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee9aa4a0>, 2329622.296393738)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d3397d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee9aa2e0>, 2329623.349270445)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d34ff10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4e881a0>, 2329623.580129982)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d7a2110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b65b00c0>, 2329623.472060148)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d37b550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b65b2c80>, 2329622.496549221)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d391890>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6855390>, 2329623.139997575)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d2833d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b68554e0>, 2329623.456099294)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d2b33d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b65b1e80>, 2329623.236014009)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d26e9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b62b1ef0>, 2329623.391841816)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d2d5050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7114210>, 2329623.860020302)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d13f790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7114b40>, 2329623.797173649)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d1504d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7086580>, 2329624.041246953)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d178bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b71149f0>, 2329624.01001531)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d2ea6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b65b1cc0>, 2329624.098153958)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d2fb910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b71163c0>, 2329624.320868138)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d1d4550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6857230>, 2329624.361859105)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d1f2690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7114c90>, 2329624.21894703)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d19a210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4562890>, 2329624.322005769)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d2b3b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7114ad0>, 2329624.349047946)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d0094d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b60ba3c0>, 2329624.63293476)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d2eecd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7115fd0>, 2329624.584832453)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d06abd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee9a8b40>, 2329624.601668513)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d08e690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b60b9be0>, 2329624.701073753)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d073a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b60b9390>, 2329624.64947487)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d1b2410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7087620>, 2329624.841511496)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d135c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b65b2430>, 2329624.935089433)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d1cff50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7002970>, 2329624.984414795)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d1e2350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7085b70>, 2329624.743638548)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cf18d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b68558d0>, 2329624.82419711)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cf346d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b70e7230>, 2329624.66823826)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d07dc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b68557f0>, 2329625.012588357)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cf53190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b60b9b00>, 2329625.167833058)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d06be90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b639e7b0>, 2329626.165195677)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cf59a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6856ac0>, 2329625.436530768)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cfd2d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b70037e0>, 2329625.484627148)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d1d4350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b506fa80>, 2329625.396160159)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ce3ba50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b506fe00>, 2329625.596835021)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ce70c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b639def0>, 2329625.252100265)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cf91390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b60bb150>, 2329625.3784259)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ce136d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b60b8ec0>, 2329625.652637083)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ce38a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b639fe00>, 2329625.539852812)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ce63310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b506c980>, 2329625.637045563)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ce8bc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b60b9e80>, 2329625.891041797)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cee2b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b639e580>, 2329625.884117036)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cef3650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5ea9400>, 2329625.876962089)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cd07b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5ea8910>, 2329626.045270038)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cd2f110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5ea8600>, 2329626.375522191)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cd8afd0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5ea9a90>, 2329626.270492521)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cebd7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5eaba10>, 2329626.059010232)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cfda490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5eaa6d0>, 2329626.168143783)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cd8a210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5eaa0b0>, 2329626.327127699)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cdc60d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b506cec0>, 2329626.637727592)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cc10ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a15035fd0>, 2329626.484954952)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cc4dc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a15035b70>, 2329626.699530293)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cf83450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a150363c0>, 2329626.706347367)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ce7d790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a150367b0>, 2329626.678969015)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cc7d8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b639c050>, 2329626.736397758)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cdd7550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5adfe00>, 2329627.075960671)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ccad810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5ade270>, 2329626.926220001)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ccc6110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6856350>, 2329626.937147039)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ccd4110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5add860>, 2329627.029683308)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ce2e750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5adeac0>, 2329627.181390593)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cb57d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5adf930>, 2329627.233723747)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cb9cbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5ea8670>, 2329627.245700034)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cbb3110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e0147c0>, 2329627.459939227)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cbd8c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e016510>, 2329628.002833969)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ce1d610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e015780>, 2329627.869066506)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ca07490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e015f60>, 2329627.164268383)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cb8d690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e0167b0>, 2329627.626431934)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cbf7990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e017ee0>, 2329627.770385713)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ca3b6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ca55940>, 2329627.870457746)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095caabbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ca55e10>, 2329627.88801203)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cee29d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ca56c80>, 2329627.971396578)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cad1450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ca56e40>, 2329628.069298606)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cadc990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ca57000>, 2329627.966450949)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095caf8750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ca54bb0>, 2329628.008812963)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cb2a710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ca577e0>, 2329627.96042315)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c91e990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ca579a0>, 2329628.005609778)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cc4dc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ca57b60>, 2329628.096233203)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c935c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5adcc90>, 2329628.317904128)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ca32250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095c959320>, 2329628.294978923)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cb3d4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ca574d0>, 2329628.558822282)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c9902d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095c95b8c0>, 2329628.669528598)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c821150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095c854050>, 2329628.708918712)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c961290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095c8541a0>, 2329628.67758717)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cadd090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095c854fa0>, 2329628.766103625)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c858390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ca54e50>, 2329628.665676354)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cb3ca50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095c855da0>, 2329628.77375543)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c94d550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095c8564a0>, 2329628.81690718)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c8b7c10>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095c95acf0>, 2329628.505727518)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c9a3dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095c8544b0>, 2329628.837518005)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c843210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d75f0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b71080c0>, 2329621.569095814)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6728a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c500210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c6c1610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c6a9dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c695cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7116a50>, 2329630.616860116)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c545c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7114360>, 2329630.47084245)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c526310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c687a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c795050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c52c510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c601f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee9a87c0>, 2329630.530481493)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c687390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7117230>, 2329630.523194072)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c795490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5fb9e10>, 2329633.604327256)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c52de50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5fb96a0>, 2329633.734065296)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c6026d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c74a590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5c5a270>, 2329630.886438517)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cadca50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c7e8550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5c58fa0>, 2329630.929016771)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c7e83d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c54fe50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095c856820>, 2329631.033548162)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ccd52d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d1f02d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095c7579a0>, 2329631.137929773)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d1f3350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cdc92d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d253e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6b5a270>, 2329631.068938712)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cdc97d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6b58fa0>, 2329631.153607894)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d2504d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cb2d4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d2096d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b65b0de0>, 2329631.157405519)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d3df490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5e27c40>, 2329631.26611081)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d20a090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d4a3750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5e27150>, 2329631.33246715)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d4a2510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d30a5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d43d650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5e24360>, 2329631.350073537)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d0b2310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d402490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c6e1490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c6f3850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c503610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c5157d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6854590>, 2329630.469158444)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c6c02d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6856270>, 2329630.695869276)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c6c3410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cbe5150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cd5a890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095c854f30>, 2329631.422263835)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c54c390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095c856430>, 2329631.047430692)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cd59f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095c7547c0>, 2329631.298059171)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d4026d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e78a970>, 2329630.35111582)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c6e1b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e684830>, 2329630.368271521)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c6f3ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e217000>, 2329630.309060893)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c503450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f0a2dd0>, 2329630.281794617)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c515dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6855010>, 2329630.538181027)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c963fd0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d024890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5e241a0>, 2329631.361296197)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c546350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d232d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4951390>, 2329631.677867281)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d231e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095caa82d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5c58520>, 2329631.944228001)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c764210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c6f0410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d7d78d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4950130>, 2329631.717443145)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d4038d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5c5b2a0>, 2329631.789366879)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d7d7550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d6bc890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d718810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b49504b0>, 2329631.906990026)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c55ef50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b49524a0>, 2329631.994046106)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d718450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d8c7890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5e25320>, 2329632.02437708)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d8c7850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d3b6b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d70d850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4950fa0>, 2329631.998025771)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d3b7f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b49508a0>, 2329631.850937919)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cfe4450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d87ce50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5e263c0>, 2329632.113883151)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d87cf10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d889ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d8436d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095c855470>, 2329632.032088059)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d88af90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee68d2b0>, 2329632.019417243)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d842010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d7a2590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d9c1810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d6078d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d5187d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5c5a4a0>, 2329631.689118324)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d518650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b65b1710>, 2329631.611403846)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d604150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5c58a60>, 2329632.363067035)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d7a25d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee68ee40>, 2329632.253605252)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5db8d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d8e0050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4952b30>, 2329632.479774988)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d929290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c734ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b67f1a90>, 2329632.590080003)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d843b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d846090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6854ad0>, 2329632.617758814)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d845fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d9a8b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d851d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5e24c90>, 2329632.703258986)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d9aa010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b68dce50>, 2329632.607221747)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d8e1090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5b82610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b7699050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b68de2e0>, 2329632.91527382)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b55da450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5d66970>, 2329632.858843512)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b769a710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d978f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a150a2650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5e27380>, 2329632.843047026)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d97b610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee7452b0>, 2329632.875609443)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a15645210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b74e9010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c874bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b67f2120>, 2329632.942339302)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b74ea850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5fbaa50>, 2329633.062938195)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6f199d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5a5ebd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d981750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d70c6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d973590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee7f3770>, 2329632.771286531)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b548b6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee68f5b0>, 2329632.108323474)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d980210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b67f1010>, 2329632.364304771)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d70c090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4951cc0>, 2329632.461374835)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d970fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5a7c310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee7f0050>, 2329633.052275496)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d70ddd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a15417090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee7f0210>, 2329633.566989927)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a15416e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b684a8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d85c0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6774ec0>, 2329633.256702434)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6849c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6776740>, 2329633.594474341)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d85c050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b471e9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b49d4550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5ed9fd0>, 2329633.175181698)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b64a4c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5c5a970>, 2329633.616898315)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b49d51d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d8454d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5ed8b40>, 2329633.599179106)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d9e1b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee658410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4aea9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b746f770>, 2329633.284981032)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d7aa390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b746d710>, 2329633.856263771)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4ae9e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee684f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b746c980>, 2329633.708597774)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b75736d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b60e1a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5845150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee7f1470>, 2329633.972535466)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b60e1290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b746df60>, 2329633.528004153)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5847750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d950c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b649f510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b68de430>, 2329633.727728007)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d951210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a15037d20>, 2329633.669176198)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b649ff10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b62faf10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5ed2610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a15037ee0>, 2329633.707229889)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cad2e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee7f1160>, 2329632.804502013)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5ed3f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a16245810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b62b2ac0>, 2329633.763796246)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b643a210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5c4ed10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b720a580>, 2329634.265658428)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5c4fc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5866450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b720a9e0>, 2329634.332360023)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5865190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4776cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6380710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b71cd5c0>, 2329634.290022516)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4775990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7209b70>, 2329682.513892456)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b56ae8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b7389350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b72289d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7209d30>, 2329682.490601834)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b60e03d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b7164610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b720bc40>, 2329682.415973288)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b7164cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b67fac50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b7071bd0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5484210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4a91d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b70efcd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b65ae5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5edb310>, 2329633.828813967)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b65aca10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5ed8c90>, 2329633.86070287)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5b32990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7208670>, 2329682.501760114)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b75eb6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6856970>, 2329682.392092059)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b488dc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b71cfee0>, 2329634.035079802)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5f34ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4952ac0>, 2329633.942435649)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4a92c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4951be0>, 2329633.983926714)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b74de9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b716f810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee83d750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b70e5e10>, 2329681.553801917)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b716f210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b74f9550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5eab230>, 2329681.579777705)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b74f8550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a14fe3d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b77f6c10>, 2329682.098908247)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b62ae350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b63aefd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee7f1630>, 2329681.78403002)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b63adf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b66f0550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b756c130>, 2329682.790623081)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b569cb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee82bfd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67c8a0>, 2329682.026041589)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee829e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4a7bd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b70e64a0>, 2329681.729469318)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b716ec10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee8124d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b59394d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b47313d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a15635bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b59e2c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee633690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67e660>, 2329682.141105322)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4a7b590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b639f2a0>, 2329681.509281278)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee812910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7208c90>, 2329681.534485426)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b593af90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b77f7150>, 2329681.644733386)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b54a3350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b77f7a10>, 2329681.578414493)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a15635710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b756cb40>, 2329683.951757529)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b62ee9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b756d550>, 2329682.853302952)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee612dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b73f6490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67c360>, 2329682.104783816)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b45ebb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee84b810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7208e50>, 2329682.176339448)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b748add0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a150b0c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b52eb4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b720b2a0>, 2329682.626010777)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a150b0690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f43d320>, 2329683.684895865)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b59fea90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b53b7290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f43e200>, 2329682.867313426)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b7295c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5ce9150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f43f9a0>, 2329683.010341665)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5ce9d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5f78190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee776210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f43c6e0>, 2329684.142187684)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5f7b690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b71cc0c0>, 2329684.122973267)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee777750>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6e32c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6e3a190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a154baf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b497bd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b66b9010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d81a510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f43d470>, 2329682.266736243)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b51cdc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f43d5c0>, 2329682.309693575)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b7541110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f43c1a0>, 2329682.395937157)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4a7b350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4e8a120>, 2329683.901473646)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6e306d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b77f6eb0>, 2329682.334768125)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4853b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f43cb40>, 2329682.592931539)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee7bf6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6b63750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b76e6290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4fd08a0>, 2329683.179242174)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6b614d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4fd0520>, 2329683.982336962)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5a86950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4a63610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f43d2b0>, 2329684.256197586)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4a62dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4dda890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e32d390>, 2329684.637822698)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4dd88d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4a191d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b482ff10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e32dc50>, 2329683.656447239)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b482ecd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b415af90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4a99590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b629e890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5b31010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5792590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4a5d3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a150485d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e32e200>, 2329683.660890319)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4a1ae50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e32cfa0>, 2329683.766250277)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4159c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67f9a0>, 2329684.157756774)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b55454d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67fee0>, 2329684.176062761)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5c2da50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f43f0e0>, 2329684.080572513)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4d26910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4fd3930>, 2329683.273556855)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee7ac750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4fd2f20>, 2329684.204009621)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4a5e2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4e8b460>, 2329684.419478385)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5afd050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee86b6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5994750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4e8a040>, 2329684.269248962)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b59d2150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67c050>, 2329684.664135509)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5996510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b48529d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e32f620>, 2329684.039029408)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee9dcc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b41dc650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b70e6190>, 2329684.547251801)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4601a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b41a9b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e44a2e0>, 2329684.526441708)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b41aaa50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f853dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e44b540>, 2329684.551436947)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f851350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f87fc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e449fd0>, 2329684.943725937)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f87c990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f93dd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e32cd70>, 2329684.951849816)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f71b1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f931910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5004dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e44b070>, 2329684.722141338)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f930350>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e448670>, 2329684.72762804)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f63b810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f732650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b499cc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e448e50>, 2329684.758954744)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f731dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4fd2660>, 2329684.87941122)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b499d610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b51e5f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67f000>, 2329684.98095783)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b51e7f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f9f0290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b46763c0>, 2329684.953028891)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4ad5410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b41003d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f95bdd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4560440>, 2329684.387726339)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f959f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e32cbb0>, 2329684.817432284)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f9f3f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b71cec80>, 2329684.003321068)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee83d1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f8bb150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4675470>, 2329685.234747859)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5c910d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a15518150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5f9e810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e32df60>, 2329685.242319868)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b48ce850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee760210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b58f0790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e55a040>, 2329685.4006438)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee762f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e55ac10>, 2329685.324258663)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4146410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b599fd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e558750>, 2329685.369453289)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4780250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b73f9590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e448750>, 2329685.454544289)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b75b8510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b72c70d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee7b4050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e558050>, 2329685.510427919)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4686010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4e8a270>, 2329685.542898541)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee7b5150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a153d7190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5c38b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b52f54d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b671a190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b51049d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4677b60>, 2329685.373613914)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5f9d550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f43fa10>, 2329684.979714201)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b67195d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e32ca60>, 2329685.057495937)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b41ddc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e44a820>, 2329685.448699452)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f78e190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b46757f0>, 2329685.548604887)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5c3b610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e55bd20>, 2329685.632889306)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b52f4250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a1551a4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4675940>, 2329685.13893379)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a1551b910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b728dfd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b72088a0>, 2329685.778492341)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b728ce90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b46857d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e32c520>, 2329685.808509357)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4743150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4639a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b63c7d20>, 2329685.732533291)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b57c5c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f625750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f738910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5ed8e50>, 2329685.973796204)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4904490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b63c4590>, 2329686.02702474)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee8a0890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e02ae10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f73b510>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b63c6510>, 2329685.96992646)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e02b290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b63c5240>, 2329686.007100487)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e2eefd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e176250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b583c3d0>, 2329686.216795699)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f6bd4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee650e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e2ab3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b583f2a0>, 2329686.132545912)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6512e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e559390>, 2329686.175474674)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e2a8e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e00f750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5566b30>, 2329686.164154913)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e00c210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e2b7b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e062910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4610c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b50045d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e4482f0>, 2329685.637164372)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e063ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e55bbd0>, 2329685.707262948)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4610710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b756d160>, 2329685.657260494)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b7095c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b55667b0>, 2329686.37080785)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e2b4d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f485150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4676cf0>, 2329686.276893743)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4af5910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5c4ac50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5565a90>, 2329686.458649987)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b699bf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e234e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b62788a0>, 2329686.747666746)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e234c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e1416d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b627ae40>, 2329686.631567168)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e142290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e205d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b63c7cb0>, 2329686.608847933)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4a3c350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f867dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b63c4ad0>, 2329686.958826265)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e059410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e073690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e44b770>, 2329686.718837343)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f54c2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e161ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee92bbd0>, 2329686.985290398)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e1639d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e1e5e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e3a4f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee92b000>, 2329687.126146023)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee660150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6278c20>, 2329686.825057097)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e3a6d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e05bd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e0718d0>\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      " {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e161610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e1d4ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5565470>, 2329686.157933292)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e059110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e44ad60>, 2329686.294565844)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e072a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e15f610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b583f070>, 2329686.387444272)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e1d5a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b627a270>, 2329686.393380799)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b57c4210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6278ad0>, 2329686.409144942)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f8ce890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b60efed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee9283d0>, 2329686.871680658)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f623fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e464d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b63c57f0>, 2329687.514074678)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e466c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e1f7310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e415c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a15521b70>, 2329687.043431524)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e1f5cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a15522350>, 2329687.604123459)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e415a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e4254d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a15523770>, 2329687.47149626)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e426f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e1d69d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a15522900>, 2329687.18464155)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e1d7010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e5a2890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e57f4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a15520600>, 2329687.399639727)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e57ff90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e333010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e39cbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e32a010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e4aa550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e222550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee9290f0>, 2329686.929899772)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e39f710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b627a740>, 2329687.335268566)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e5a1650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee92baf0>, 2329686.875379206)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e32ac10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e333190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e286e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee928d00>, 2329686.950136573)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e4e22d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e55b930>, 2329687.77505677)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e34c5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee9280c0>, 2329687.038411536)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e223750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee929e80>, 2329686.972137415)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e331410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a155210f0>, 2329686.992531493)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e499510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e5b8e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4150b40>, 2329687.627368361)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f73b0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e668790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b41535b0>, 2329687.78013661)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e669a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e3a76d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e32e580>, 2329687.713789916)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e11d4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e56a950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e605c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4151630>, 2329687.870129643)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e569c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b41529e0>, 2329687.866494018)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4741c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e513950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4151da0>, 2329687.718658257)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e512250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e511190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f797770>, 2329687.942734855)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e513ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e7a4390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f795390>, 2329687.863859797)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e7f87d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e57ba50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f7948a0>, 2329688.739607322)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e579b90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e705110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b41520b0>, 2329688.096559551)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e704e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e8beb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4d24fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e6bb890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e674c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e62ff90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e6d4ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4153380>, 2329687.687534082)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e557350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a15521780>, 2329687.570239317)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e6d4b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f43e6d0>, 2329687.743083179)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e674cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a15521ef0>, 2329688.047109103)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e8bc310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b41500c0>, 2329687.663295484)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e6b9390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4150520>, 2329687.693218111)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e62c610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e7b9c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f794600>, 2329688.314193067)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e6b9450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e85d090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e828750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f797a80>, 2329688.318146519)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e85ce50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f7974d0>, 2329688.39593302)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e82a590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e9db6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a15520360>, 2329688.425808738)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e9d9d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e9d19d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e9a5190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f8b03d0>, 2329688.469759134)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e9dbfd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f8b3a10>, 2329688.711403014)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e9a78d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e987810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e962890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee928c90>, 2329688.529418039)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e5add90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f8b2820>, 2329688.712863732)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e9754d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e942f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f8b2cf0>, 2329688.740758008)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e941a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e88c890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e92b650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f8b0130>, 2329688.483045305)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e88c8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f7954e0>, 2329688.931064772)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e928790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095eaecc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095eae5a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f8b39a0>, 2329688.897129642)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095eaecf50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ea7d010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5df52b0>, 2329688.753102533)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ea5ed90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e5124d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5df42f0>, 2329689.001272574)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095eae4e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e8b2790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ea54190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f8b3620>, 2329688.741865451)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e9b75d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5df6430>, 2329688.725917348)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ea72950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e559da0>, 2329688.715899164)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ea55fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e86f090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4f97380>, 2329689.257979156)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e86f510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095eb38bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a15521f60>, 2329689.171341275)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095eb3b390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e716a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f8b2970>, 2329689.247653649)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e715250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ec95150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4f95e10>, 2329689.204154153)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ec94050>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ec96350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a155224a0>, 2329689.374166956)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ec94e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095eba3010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4f962e0>, 2329689.321149622)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f636710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e960cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095eb62910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ec17a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453d010>, 2329689.536806457)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e142f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453ec80>, 2329690.785227181)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095eb63210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453f150>, 2329689.611033822)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ec16110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ec00050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ebd1e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ebb0ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ebebe10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a15521c50>, 2329688.947287727)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ebe9f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ca55320>, 2329688.876716971)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ebbfe90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453f7e0>, 2329689.555205795)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ec01350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4f94830>, 2329689.0084246)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ebb3e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095edd8cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453ff50>, 2329689.590274733)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095eddbb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ec5a710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e809550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ee9c350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5df7e70>, 2329689.725191653)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ed05c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453e190>, 2329691.3072456)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ed0d510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ec64b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453e7b0>, 2329689.862956567)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ec64c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095eb50d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7193b60>, 2329690.763964672)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095eb524d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095efef310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453d550>, 2329691.011248662)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ed41f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ee74d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b46778c0>, 2329689.965145022)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ee759d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ee0d250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ef8eb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095edd21d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f352c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095edb8810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee92a120>, 2329689.685203283)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ec5be90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095eed3f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453d860>, 2329689.447806068)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e69ac10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5df6740>, 2329690.993079334)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ee0c3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ee178d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095efd5d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453ef90>, 2329689.603329479)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095edd1a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f9ec6e0>, 2329690.177205912)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ef8c4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453c280>, 2329690.594245991)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095edbbc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b71924a0>, 2329690.652815923)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095eed2650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453d1d0>, 2329691.05028953)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ee17c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453fd20>, 2329691.108036446)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095efd5c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095eff7a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4f954e0>, 2329690.283157834)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095eff5e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095efe6b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7192d60>, 2329690.50589101)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095efe4f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f08c2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f9eef90>, 2329690.501828638)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f08e810>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f00b790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f1ce590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f9ecec0>, 2329691.136838768)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f1cc110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f19ecd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095eea8b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ef13a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f0f8410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f0b3b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f0a7cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f8b2660>, 2329690.743754955)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ef13a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f8b3460>, 2329690.866009298)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f301990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f60ff10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5912c80>, 2329691.270505058)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f19ca90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f9efd20>, 2329690.524508744)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f0a7b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7193a80>, 2329690.561671741)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f60ead0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f47ecd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4f97a10>, 2329690.132953613)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095eeaa390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453d4e0>, 2329690.477423986)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f0faa90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f9ec670>, 2329690.608524412)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f0b0850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f9eeba0>, 2329690.798532165)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f47f8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f26db50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f91b3f0>, 2329691.379481059)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f6cdb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f2bea50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f386c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b446c280>, 2329691.350809208)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f2bfb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f9efee0>, 2329692.464279032)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f3850d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f10f1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f19a890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b744d390>, 2329691.556992812)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f19b310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f114f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f4cc4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f4c5390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b446c750>, 2329691.609293538)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f4cdfd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b446e190>, 2329691.55871724)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f4c4610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f977e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f308b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f9ecb40>, 2329691.687150724)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f976fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b446da20>, 2329691.866829009)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f308cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f4b5e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f17d4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f3c7c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f58c390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095efaa210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b71923c0>, 2329692.436724112)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f10f990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b446de80>, 2329691.553070469)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f114ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f91a7b0>, 2329691.242493048)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f58ef10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7193af0>, 2329691.257221563)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f17eed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f91a580>, 2329691.296986237)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f3c7190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f032890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f918280>, 2329691.310914877)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ef8c790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f91ac10>, 2329692.938143554)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f004390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453f620>, 2329691.369657483)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f5c4790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c567890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e2e9400>, 2329692.058090146)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c567bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f517050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bd3d150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e2eaba0>, 2329692.224004361)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ef8dc50>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bd4ea90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e2ea9e0>, 2329692.045328786)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bd4e890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f4e7750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e2e8f30>, 2329692.000732203)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f4e4e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f395390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f1af0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e2e9a90>, 2329691.986048777)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f15ec10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e2ea7b0>, 2329692.313890793)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f1aee10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c55a8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bd97450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e2ea740>, 2329692.445563366)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c55bbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e2ea350>, 2329692.262824648)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bd97250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bdc5650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e2e9240>, 2329692.408940863)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bdc5450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bdd3590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c516e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c533950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c549990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bd20e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e2ea970>, 2329692.15559834)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bd3cf50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b72b2200>, 2329691.907272374)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c517310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e2e8360>, 2329692.452716462)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bdd3450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e2ebd90>, 2329692.03577805)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f3f3810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e2eb690>, 2329691.962305177)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c549390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e2eb850>, 2329693.08183666)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bd20c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b73c64d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b72b3bd0>, 2329692.909256736)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee986350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bdb4a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bc3f590>\n",
      "Evaluating workflow:  60%|██████    | 1802/3000 [06:12<01:17, 15.55it/s]Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e2ebe00>, 2329693.085613083)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bdb4b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e684210>, 2329693.051659025)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bc3dc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bde6b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b72b1550>, 2329692.950875263)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bde6190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bc56c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bc65f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e687690>, 2329692.747059447)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bdb59d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e685ef0>, 2329692.793858405)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bc65d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bc6fe50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e686ac0>, 2329692.915142587)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bc6fd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bc7fa90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e8ad860>, 2329693.838551791)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bc7fed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bdc5e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bcbe190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e8ada20>, 2329693.110635303)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bde7710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e8acad0>, 2329693.007833477)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bcbdf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bcc4990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e8afe00>, 2329693.059615684)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bcc6dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bc7ead0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b446eeb0>, 2329693.169080991)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bc7e5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bcaaa10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bb05590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bd4c390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bc184d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e685da0>, 2329692.68051428)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ec391d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453faf0>, 2329692.710586344)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bd4ea50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e8adcc0>, 2329694.229167035)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bcaa110>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e8af700>, 2329693.344988156)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bb05a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bca2e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e8afb60>, 2329693.293705465)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bca2b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bdd8150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bb85d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e8ae510>, 2329693.671747094)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bdd3250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e9bb070>, 2329693.568024657)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bb85b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bb94410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e685e10>, 2329693.497884459)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bc6ef50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bba3c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e9bae40>, 2329693.680902771)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bba3ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bbc6a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e9b8b40>, 2329693.870620228)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bbc6890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bbe5f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e685390>, 2329693.832791035)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bbe5e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094ba15f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e9b9fd0>, 2329693.786380505)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094ba15e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bb616d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094ba35f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e2ea2e0>, 2329694.182600242)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bb61d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eae1780>, 2329693.82686095)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094ba35d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bd41c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094ba589d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bb1cb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bbc4990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094ba09190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b72b08a0>, 2329693.788184898)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bc25050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eae0980>, 2329693.87468323)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094ba59090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e8ac360>, 2329694.238779893)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bb1f410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b446c210>, 2329741.772795027)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bbc5050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e9b9710>, 2329742.99249763)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094ba08bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bacc290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bae5bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eae1240>, 2329694.418197493)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bae4050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094ba73310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b90f050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e2e9ef0>, 2329694.484726968)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094ba730d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eb1e900>, 2329741.861779306)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b90dc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b930e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eb1d860>, 2329694.461739287)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b931510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094baad8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b946690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eb1c590>, 2329694.762724114)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bb05f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b94b550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eb1d160>, 2329694.541474575)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b94b410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094ba16e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094ba4c410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e8aea50>, 2329694.642893078)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094ba15d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eb1ee40>, 2329742.75442902)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b973fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bb36c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b989950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bc7ccd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094ba5a150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eae1f60>, 2329694.38644538)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bacc8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eae1b70>, 2329693.999009494)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094ba67450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094ba93dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eae3ee0>, 2329693.994505978)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094ba93c50>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eb1f7e0>, 2329742.676917171)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b946d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eb1d390>, 2329741.824656247)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b962210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eb1c130>, 2329742.625005389)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bbb9990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b59125f0>, 2329693.896394185)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bc7d090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bdd30d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e2e9860>, 2329694.387696727)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bdd3e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b9bff10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ebf7d20>, 2329742.285551771)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b9bff90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094ba72fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b9b7ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b9d9310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b9fd4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094ba65690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b910250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eae33f0>, 2329741.998414739)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094ba0ad50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ebf4de0>, 2329741.832915919)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b9b7dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e686580>, 2329742.971144468)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b9db790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ebf6740>, 2329744.324371666)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094ba95510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eb1cb40>, 2329744.191507935)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094ba66b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ebf4910>, 2329742.136029188)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b83db50>\n",
      "Evaluating workflow:  60%|██████    | 1805/3000 [06:12<01:09, 17.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1808/3000 [06:12<01:11, 16.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|██████    | 1813/3000 [06:13<01:30, 13.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1818/3000 [06:13<01:05, 18.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1827/3000 [06:13<00:49, 23.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1830/3000 [06:14<01:16, 15.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████▏   | 1838/3000 [06:14<00:54, 21.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████▏   | 1842/3000 [06:14<00:47, 24.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1852/3000 [06:15<00:49, 23.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1856/3000 [06:15<00:48, 23.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1860/3000 [06:15<00:50, 22.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1863/3000 [06:15<00:57, 19.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1865/3000 [07:02<1:17:32,  4.10s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1868/3000 [07:02<50:55,  2.70s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1873/3000 [07:02<26:43,  1.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1878/3000 [07:03<14:34,  1.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1884/3000 [07:03<07:31,  2.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1887/3000 [07:03<05:32,  3.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1891/3000 [07:03<04:07,  4.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1894/3000 [07:04<03:37,  5.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1899/3000 [07:04<03:05,  5.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▎   | 1906/3000 [07:05<01:49, 10.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▎   | 1909/3000 [07:05<01:30, 11.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1914/3000 [07:05<01:31, 11.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1921/3000 [07:06<01:03, 17.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1928/3000 [07:06<00:48, 22.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1931/3000 [07:06<00:59, 17.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1936/3000 [07:06<00:45, 23.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▍   | 1942/3000 [07:06<00:47, 22.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1945/3000 [07:07<00:47, 22.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 1952/3000 [07:07<00:47, 22.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 1959/3000 [07:07<00:47, 22.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1963/3000 [07:07<00:48, 21.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1970/3000 [07:08<00:45, 22.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1977/3000 [07:08<00:43, 23.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1980/3000 [07:08<00:41, 24.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1987/3000 [07:09<00:46, 21.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▋   | 1994/3000 [07:09<00:44, 22.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 1997/3000 [07:09<00:45, 22.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 2003/3000 [07:09<00:46, 21.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2006/3000 [07:09<00:42, 23.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2009/3000 [07:10<00:51, 19.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 2015/3000 [07:10<00:56, 17.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 2022/3000 [07:10<00:50, 19.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2028/3000 [07:11<00:46, 20.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2036/3000 [07:11<00:44, 21.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2039/3000 [07:11<00:43, 21.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2044/3000 [07:11<00:36, 26.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2047/3000 [07:12<00:50, 18.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2053/3000 [07:12<00:46, 20.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▊   | 2061/3000 [07:12<00:39, 23.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 2068/3000 [07:13<00:47, 19.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2071/3000 [07:13<00:45, 20.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2076/3000 [07:13<00:34, 26.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2080/3000 [07:13<00:37, 24.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|██████▉   | 2086/3000 [07:13<00:50, 18.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|██████▉   | 2096/3000 [07:14<00:34, 26.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 2099/3000 [07:14<00:36, 24.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|███████   | 2105/3000 [07:14<00:41, 21.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 2109/3000 [07:14<00:39, 22.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 2114/3000 [07:14<00:33, 26.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2117/3000 [07:15<00:45, 19.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 2124/3000 [07:15<00:43, 19.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2127/3000 [07:15<00:48, 18.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2130/3000 [07:16<01:05, 13.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2132/3000 [07:16<01:23, 10.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2132/3000 [07:29<01:23, 10.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 10:01:39.377\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert analyst of perturbation datasets. For K562 cells subjected to perturbation of DDOST, does the expression profile of TNFRSF10B indicate a significant change relative to control conditions? Reply strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are an expert analyst of perturbation datasets. For K562 cells subjected to perturbation of DDOST, does the expression profile of TNFRSF10B indicate a significant change relative to control conditions? Reply strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 3867}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 2134/3000 [08:02<1:12:27,  5.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2137/3000 [08:02<43:39,  3.03s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████▏  | 2141/3000 [08:03<23:44,  1.66s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2145/3000 [08:03<12:24,  1.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2148/3000 [08:03<07:50,  1.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2153/3000 [08:03<04:13,  3.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2159/3000 [08:04<02:18,  6.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2162/3000 [08:04<01:41,  8.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2165/3000 [08:05<02:35,  5.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2170/3000 [08:05<01:49,  7.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2174/3000 [08:05<01:22,  9.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 2183/3000 [08:06<00:38, 21.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 2190/3000 [08:06<00:35, 22.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2193/3000 [08:06<00:46, 17.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2196/3000 [08:06<00:46, 17.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2202/3000 [08:06<00:31, 25.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▎  | 2210/3000 [08:07<00:33, 23.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2213/3000 [08:07<00:43, 18.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2220/3000 [08:07<00:32, 23.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 2227/3000 [08:08<00:33, 23.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2230/3000 [08:08<00:36, 21.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▍  | 2237/3000 [08:08<00:35, 21.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2244/3000 [08:08<00:26, 28.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2247/3000 [08:08<00:29, 25.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▌  | 2253/3000 [08:09<00:37, 20.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 2258/3000 [08:09<00:30, 24.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 2265/3000 [08:09<00:29, 24.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2268/3000 [08:09<00:35, 20.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2272/3000 [08:10<00:31, 23.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 2278/3000 [08:10<00:33, 21.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2283/3000 [08:10<00:25, 27.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2287/3000 [08:10<00:38, 18.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 2296/3000 [08:11<00:36, 19.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 2302/3000 [08:11<00:31, 22.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2305/3000 [08:11<00:31, 21.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2308/3000 [08:12<00:42, 16.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2316/3000 [08:12<00:25, 26.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2320/3000 [08:12<00:32, 20.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2323/3000 [08:12<00:39, 17.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2329/3000 [08:12<00:33, 20.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2336/3000 [08:13<00:30, 21.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2340/3000 [08:13<00:28, 22.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2343/3000 [08:13<00:37, 17.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2347/3000 [08:13<00:32, 20.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2353/3000 [08:14<00:34, 18.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▊  | 2356/3000 [08:14<00:31, 20.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 2363/3000 [08:14<00:32, 19.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2366/3000 [08:14<00:35, 18.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2369/3000 [08:15<00:36, 17.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2372/3000 [08:15<00:31, 19.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2377/3000 [08:15<00:31, 19.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b9fe910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ec0eba0>, 2329743.340390386)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b9ff690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094ba864d0>\n",
      "Evaluating workflow:  79%|███████▉  | 2380/3000 [08:16<00:58, 10.64it/s]Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ec0cde0>, 2329742.291634658)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094ba86dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b8a55d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ec0c9f0>, 2329742.230736613)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b8a5450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b875390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bac8c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ec0cc90>, 2329742.229393679)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b875250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ec0f310>, 2329742.422399426)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b9fd0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b9e23d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eb1d940>, 2329742.615054763)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b9e08d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b7aff50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ef96eb0>, 2329742.963946452)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b783850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b753c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b7711d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b78d3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b7afa10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ed3fa10>, 2329742.716745393)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b753b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ed3dc50>, 2329742.575045708)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b770e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b716150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b73bd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ed3cc90>, 2329742.564888451)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b715350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ed3edd0>, 2329742.734517792)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b73bbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ed3e900>, 2329742.750449292)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b78db90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ed3fb60>, 2329742.830194114)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b7afe90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b64bc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ef956a0>, 2329744.447425808)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b783650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b6d0410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f0a3230>, 2329744.706697269)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b6d0ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b702510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b649cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bac84d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b6700d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b6947d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b612450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b6f24d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b508490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094ba364d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eb1f4d0>, 2329743.412294007)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094ba37190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b682ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ed3e820>, 2329744.47153255)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b702910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ef96e40>, 2329743.317012697)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b649ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ef94440>, 2329744.558440288)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b64b590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ef94210>, 2329744.543348741)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b610810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ed3f5b0>, 2329744.56484027)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b694650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ef94130>, 2329744.806946768)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b6129d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ed3dda0>, 2329744.974808504)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b6f1890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eb1f310>, 2329745.064204819)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bac8c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f0a30e0>, 2329744.874089242)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b52ab10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b844510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b548650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ef94280>, 2329744.559624174)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b69f3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ed3fa80>, 2329744.925956527)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b548a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b529f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f201860>, 2329744.673352412)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b773a50>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b57d710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f32da20>, 2329745.188450747)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b57d3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b57c110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b5912d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f2019b0>, 2329744.194925085)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b57c490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f203cb0>, 2329744.166009092)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b5910d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b5ec090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f203c40>, 2329745.427755946)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b5ec0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b4497d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f32def0>, 2329745.5272149)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b4c0450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b445a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b570b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b4e0750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f32f000>, 2329745.350565511)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b571090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b4b2350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f0a0a60>, 2329745.492321365)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b4b2a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f1d7380>, 2329746.055540271)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b44a650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f32ca60>, 2329745.497367003)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b4e0e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b3e36d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f1d6040>, 2329746.191399001)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b330390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b3b9b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f5596a0>, 2329746.310010011)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b3ba190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b28e7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f5599b0>, 2329746.397979823)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b28e690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b2b2890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f2e6740>, 2329746.501740449)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b219890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b2f0210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b3cebd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b218410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b240c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b355bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b363090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f0a2740>, 2329746.104346373)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b3e9c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f558830>, 2329746.235708558)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b363110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f1d6d60>, 2329746.733793531)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b2f04d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f1d5860>, 2329745.948577522)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b3cea10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f200c20>, 2329746.228036156)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b4f3510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f1d78c0>, 2329746.314664047)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b240b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b271f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f2e49f0>, 2329746.679437701)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b270dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b4fb6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f2e6350>, 2329746.740712649)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b257f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b344190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f2e5860>, 2329747.44063188)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b105910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b278650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e215fd0>, 2329747.210165051)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b278510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b174310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e217af0>, 2329747.765835804)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b257c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b084990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f0a3bd0>, 2329747.536158888)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b084e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094afade50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095c631940>, 2329748.422012326)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094afaf190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094af4f490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095c6325f0>, 2329748.866477116)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094af4fc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b1f3ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094af76c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094af95210>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094afc2f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f1d63c0>, 2329748.00874833)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094af77290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e788d00>, 2329748.006640146)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094af950d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094afc1990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e789010>, 2329748.09472453)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b04fad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e78b8c0>, 2329748.097105076)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094afc2110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e7882f0>, 2329748.08971594)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094afc2390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094aeba2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094aeb0670>, 2329748.66281966)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094aeba210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094ad48990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094aeb2ba0>, 2329748.918974798)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094ad49010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094af60150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094aeb2f20>, 2329748.880270596)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b02e450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094aef5b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094aeb00c0>, 2329749.069976308)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094ae37f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094aee8ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094ad4bcd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094ad70f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094aeb1a20>, 2329748.820449217)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094aee36d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094aeb2d60>, 2329748.891529926)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094ad4bb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094aeb3700>, 2329748.814208383)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094ad71650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094ad05510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094adbb000>, 2329749.497444318)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094ad057d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094ad68f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f2e5d30>, 2329749.641722267)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094ad68b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094ac2d9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094adbadd0>, 2329749.380250009)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094aed4650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094abb9990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094ac9a040>, 2329751.359100849)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094afadf50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094acba1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094aba3d90>, 2329750.651700816)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094aa5cad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094ab6d9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094ab80bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095c6317f0>, 2329750.070513345)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b041b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094aba01a0>, 2329750.080800121)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094ab81250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094aaab8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094aa88e50>, 2329751.175081068)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094aaabb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094abff150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094aa89b00>, 2329750.746126973)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094aad3c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094aaa8110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094aa8a900>, 2329751.127806654)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094aaa8210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094aa15610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094aaa9910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094aae36d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094aafccd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094a92f2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094aa89da0>, 2329750.886875528)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b3511d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094aa8a740>, 2329750.896423706)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094aafcad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094aba1240>, 2329750.973582173)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094a92ca90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094a975ef0>, 2329751.517722453)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094a9b2690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094aa88c90>, 2329750.677800424)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094aaa97d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094a953b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094a804090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094a976660>, 2329752.054818789)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094a953cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b744e6d0>, 2329754.766412516)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094a804c50>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094a884850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094a88c4b0>, 2329752.701934878)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094a884650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094a8924d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094a88cb40>, 2329752.372156374)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094a892490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094a8c2e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094a88d940>, 2329752.46417723)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094a8c2c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094a8e3950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094aa8b380>, 2329752.55647335)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094a8e3750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094a989a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094a977540>, 2329752.668261805)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094a8538d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094a7280d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094a88cd00>, 2329752.361900838)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094a7288d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094a847510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094a73fd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094a98ab90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094a82c610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094a82de50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094a8467d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094aa54950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094aba24a0>, 2329752.083841708)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094a98ad50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094a976f20>, 2329752.071927981)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094a823a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094a88c2f0>, 2329752.060228468)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094aa55590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094a9770e0>, 2329752.061274688)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094a82e190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094aba14e0>, 2329752.068913949)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094a8465d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094a88f7e0>, 2329752.592227538)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094a847490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094a88fa80>, 2329752.469094982)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094a73fc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094a8b93d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b7d0ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b7f0f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b70a690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b61b610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b6336d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b56f810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b5d5150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b5eef90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b5bf190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b5a5290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b5c13d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b47d710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b42ec50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b469bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b47fc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b4f8050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b312cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b326110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b49bd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b362490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b385990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b84e110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b43e350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b3f21d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b257c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b38db10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b28c690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b3bba10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b114050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b13b8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b15f910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b175250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b18ad90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b1a9d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b1b36d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b1c5f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b1d7d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b1f3a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b015a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b176e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b1fb6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b06f510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b0992d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b0db6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b0f5e90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b0fe4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094af1c6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b0b2fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094af4f0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b005bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094af404d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094af60450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094afd8cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094aff8150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094ae17210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094ae46910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b1f2650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094aeaa9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094af3bf50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094aecba90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094ae63550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094aff8b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b02fa90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094ad57b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094ada0850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b0a2c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094ade6490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094adfb2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094ac49350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094ac6a2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094ac81910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094ac75550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094aca13d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094adc9910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094ac2f2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094ac24310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094acd93d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094adfea10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094acefc50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094acfb750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094ade71d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094abe4f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094aa15250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094aa2da10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094ad946d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094aa56a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094aa79810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094abc27d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094ab5d490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094abfd790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094aa92150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094aebd690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094aa568d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094a9b3750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094a9a2250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094a9d95d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094a9e6010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094a86c090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094a975d30>, 2329752.0661118)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094a9d9450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094ac98600>, 2329752.05738101)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094a9a2050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094a88ff50>, 2329752.772347448)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094a8ba050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e8aeeb0>, 2329744.141512984)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b7d0c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ef96660>, 2329743.126943895)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b7f2f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ef95160>, 2329743.0726338)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b70a8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ebf4ad0>, 2329744.276786814)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b61b410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f0a1390>, 2329743.764594244)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b6332d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f0a2120>, 2329745.169081675)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b56f6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f201780>, 2329745.220512672)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b5d4f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f2002f0>, 2329745.535592933)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b5eedd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f32fd90>, 2329745.286987761)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b5bcd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f32d8d0>, 2329745.216728562)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b5a5490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ec0d8d0>, 2329745.2712476)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b6f36d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f0a2040>, 2329745.363254051)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b47d550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f32e200>, 2329745.145009303)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b42ea50>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f0a1470>, 2329745.31029022)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b469a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f32ec80>, 2329745.447570215)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b47ef50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ec0d710>, 2329745.65397606)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b4f82d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f1d5630>, 2329745.637203107)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b313390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f1d43d0>, 2329745.724348309)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b326850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f203070>, 2329746.006891386)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b5ed950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e687bd0>, 2329745.99634169)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b633350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f0a1a20>, 2329745.878179276)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b386710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f1d4750>, 2329745.986335882)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b5d6950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f1d67b0>, 2329746.165658004)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b43eb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f55ba80>, 2329746.320091257)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b3f2910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f55b460>, 2329746.41905827)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b257ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f55ad60>, 2329746.383722267)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b38e090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f202970>, 2329746.503207454)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b28cd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f2e6f90>, 2329746.64868182)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b4b0650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f202c10>, 2329746.894877085)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b115c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f32fbd0>, 2329747.010449152)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b13bf10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f2e4210>, 2329746.998291962)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b15d110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f559fd0>, 2329747.156157706)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b2cd790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e217310>, 2329747.046336827)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b18ab90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f2e5da0>, 2329747.290294275)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b1ab510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e2173f0>, 2329747.059055673)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b1b3710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e2159b0>, 2329747.112472666)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b1c7fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e2175b0>, 2329747.169077211)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b1d7c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f2e7310>, 2329747.111174128)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b1f38d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e216660>, 2329747.366772902)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b015890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e215d30>, 2329747.212157055)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b177150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e214ec0>, 2329747.222074346)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b1f9b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ee10210>, 2329747.514888898)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b06f390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ee10ad0>, 2329747.557636441)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b099190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f559390>, 2329747.827497274)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b0db4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ee11ef0>, 2329747.944525259)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b0f5d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ee111d0>, 2329748.146306783)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b0fec10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ee126d0>, 2329747.819976653)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094af1c4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e789080>, 2329748.0351716)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b0b2790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f2e4de0>, 2329748.084892958)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094af4dc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f55baf0>, 2329747.617344646)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b004d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e78a580>, 2329748.093595605)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094af0fc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e788440>, 2329748.108032526)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094af57a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e78bcb0>, 2329748.362522536)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094afd9110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f2e7070>, 2329748.617626147)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094aff87d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e214f30>, 2329748.544678375)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094ae170d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ee106e0>, 2329748.500604687)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094ae46710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095c6301a0>, 2329748.342843063)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094ae0a710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094aeb0130>, 2329748.712074918)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094aeaa890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094aeb03d0>, 2329748.642802636)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094af3a7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095c630d00>, 2329748.900248187)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094aecb910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094aeb1080>, 2329748.886677978)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094aebfe10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094aeb26d0>, 2329748.871513194)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094afc0bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094aeb2970>, 2329748.876466734)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094ae6ec10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094aeb30e0>, 2329748.905629837)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094ad55c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094adb8c20>, 2329749.023483735)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094ada0650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f2e5160>, 2329749.225988526)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094af71bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094adb94e0>, 2329749.273726275)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094ade6290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094adb9fd0>, 2329749.454047015)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094adf90d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094adb87c0>, 2329749.430353561)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094af22050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094adbb690>, 2329749.771823247)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094ac6a990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095c631e80>, 2329749.666671018)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094ac81d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094ac98050>, 2329749.687244607)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094ac75850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e78b690>, 2329749.850803172)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094aca11d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e788ad0>, 2329749.878880109)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094adc9550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094ac9b380>, 2329750.699182375)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094ac39ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094ac99390>, 2329749.69337615)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094acb36d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094aeb1ef0>, 2329749.661181432)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094acd91d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094ac999b0>, 2329749.588909169)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094adfed90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094ac99b70>, 2329749.707629177)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094acefb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094ac99e10>, 2329749.929392875)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094acfb610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094aeb14e0>, 2329750.863904812)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094ade6c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094adba970>, 2329750.420852926)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094abe4d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094aba2740>, 2329750.617784714)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094aa155d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094aba2dd0>, 2329750.634229301)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094aa2d8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094aba3540>, 2329750.62900425)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094aa43f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094adb97f0>, 2329750.755017824)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094aa56950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094aa88050>, 2329750.811013709)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094aa79fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094adbbcb0>, 2329750.402118899)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094abce210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094aba1b70>, 2329750.209266216)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094acda690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e78b0e0>, 2329750.454026856)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094abfdb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094aa88a60>, 2329750.722875614)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094aa91f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e78bbd0>, 2329751.297781837)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094a920dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e78a040>, 2329751.334432324)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094aca1ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094a975a20>, 2329751.31999965)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094a9b3590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094a974050>, 2329751.450120414)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094a9e5ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094a88c050>, 2329752.075965346)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094a86c410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b3cc150>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b104d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094ada29d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f202eb0>, 2329746.20083619)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b71c710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e214050>, 2329747.305161008)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b104290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e788590>, 2329749.945387833)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094ab486d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094a65a190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094a6b5fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b720a200>, 2329802.246606343)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094a65ae50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f8b1e80>, 2329754.087384841)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094a69c390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094a7df390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094a7c32d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094a7a6390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eb1da90>, 2329753.353457538)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094a7dee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eb1ec80>, 2329753.474702269)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094a6ac210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eb1ef90>, 2329753.185923018)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094a7a6ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094a8f5ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eb1ee40>, 2329753.310150036)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094a730710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b0b9810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b72b26d0>, 2329753.854163673)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b0ba0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b14cb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7191550>, 2329753.837440916)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b14f990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b28fcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094a9759b0>, 2329754.745332406)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094a6cee90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094ac53250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b47dcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b71920b0>, 2329754.008244179)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094ac50a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b72b2040>, 2329753.996042272)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b47e950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b54a850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b783ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5df4de0>, 2329754.198944337)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b54b950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5df5940>, 2329754.487688823)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b781690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b6d0790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094a63a190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094a652850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094a642190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094a6ba190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094a6af090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094a68f790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094a731310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094a660110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094a830a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094a892bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094ab5cc50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094a60fd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eb1d390>, 2329753.366166269)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094ab1ee10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eb1e430>, 2329753.348015524)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094a6807d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ec0fe70>, 2329753.385029833)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094a799750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5911630>, 2329753.482787824)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094a891c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094a8fc550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094a977150>, 2329753.582910712)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094aa7a6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094a974360>, 2329754.229504295)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b6d1490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b446ea50>, 2329753.043713533)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094a63ae50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b59104b0>, 2329753.195406785)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094a652590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5913150>, 2329753.009179272)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094a6ba7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094aeb12b0>, 2329753.243527299)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094a6b65d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eb1e5f0>, 2329753.271573314)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094a68fdd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eb1fd90>, 2329753.244876134)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094a731550>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ec0d1d0>, 2329753.337243321)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094a830150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5910d70>, 2329753.563416773)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094ac2e290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b886250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f8b1ef0>, 2329754.685089652)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b886790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094a7ad050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094a691a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b41523c0>, 2329754.588860032)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b14c2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5df7c40>, 2329754.818108843)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094a693b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b619410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b982790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4152a50>, 2329754.786855886)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b619250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7191d30>, 2329754.812976188)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b9810d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b80f250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b8ccdd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5df7310>, 2329755.034728368)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b80f210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5df7bd0>, 2329754.853237507)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b8ccbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b9b4190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094aed5750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b538090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b427210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094a73f7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b728850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b8bfed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f8b2eb0>, 2329754.18326928)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094aed5110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b668b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eb1c8a0>, 2329754.221945274)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b7289d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f8b0980>, 2329754.230757903)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b425010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f8b08a0>, 2329754.264045919)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b66bf90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b744cc20>, 2329754.958409785)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b9b4390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f8b14e0>, 2329754.036623557)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b538510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f8b3700>, 2329754.097157031)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094ae174d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f8b06e0>, 2329754.332444209)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b8bf490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094a730b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b744f690>, 2329754.053773979)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094a731b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b6d0310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4152ba0>, 2329754.828266639)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b72d190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b6b4f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b744e510>, 2329754.951852281)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b6b5b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bae76d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f8b2580>, 2329755.20320684)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bae5ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b887050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b627ae40>, 2329755.061825065)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b886c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b1cd510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5564520>, 2329755.223606022)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094a9ffe90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b946610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5564600>, 2329755.424078918)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b9450d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bb850d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b744e270>, 2329802.93196714)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bb87d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094ba27490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b9217d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67d550>, 2329802.541859643)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094ba259d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67f070>, 2329802.718450144)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b922f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094ba34950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b62788a0>, 2329802.795389972)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094ba34610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bba29d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094ba85e10>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4152f90>, 2329802.719949843)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bbcdc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b9a15d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b923910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b8a5090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bacb7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4153310>, 2329754.942477907)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b90efd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ec0d0f0>, 2329755.231847297)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b8b04d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6279320>, 2329755.010822651)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b9a1390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b627acf0>, 2329755.09355725)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bac9710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67dbe0>, 2329802.525960305)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094a64dc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67d1d0>, 2329802.868383565)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b71e650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bc5e390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67d8d0>, 2329801.950331129)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bc5e0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bcade10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b756e3c0>, 2329803.222778938)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094ba29b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bdb4b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b756e430>, 2329803.202844129)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bdb6e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bd439d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b756f8c0>, 2329803.775749843)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bd40910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bb4f110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bc09950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bc25210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bc64810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bc35d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b886b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bc95d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b627b9a0>, 2329801.818231841)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bb4f250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67c3d0>, 2329802.720992632)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bc09390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67def0>, 2329802.195841629)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bc26150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67d7f0>, 2329803.178139157)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094ba4c750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094aa89a20>, 2329801.941282426)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bc34e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b756e6d0>, 2329803.097536019)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b886d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b756d0f0>, 2329802.172057337)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bc95c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bb75090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b756d2b0>, 2329801.963181303)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bb76fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f0d36d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b71cd1d0>, 2329803.27380663)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f0d2010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f268050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b71cf770>, 2329803.249368047)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f268a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094a9eda50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f2f1e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67fcb0>, 2329804.709525573)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b5324d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee7f31c0>, 2329804.61261218)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f2f07d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ef43310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ef85750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67e660>, 2329804.59890439)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ef41fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee7f1a90>, 2329804.633230506)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ef86050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095efaca50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee745940>, 2329803.541962086)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095efaded0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095efca910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f4b54d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f915590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bcf1c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f4d9fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bc94fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094b5f5d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095efc3090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b71cd240>, 2329802.989182778)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094a672f90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6278280>, 2329803.043612168)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f4b5e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b71cf9a0>, 2329803.006718994)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bd4e810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7191160>, 2329803.044811216)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f3a6890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b746f2a0>, 2329803.123275922)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095efc1d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5fb80c0>, 2329804.775796032)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095efc9810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b72092b0>, 2329805.004893135)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f917b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b71cc750>, 2329802.746351023)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f4da090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095efeca90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5fba890>, 2329803.918812227)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095efeeb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ee7af50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee7456a0>, 2329806.270814766)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bd3c8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095efed950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b744d400>, 2329805.190666981)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095efee8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e6ca8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ee40390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ee2aad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f08e810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f40a4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ed78890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ed424d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ef8cb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095eb6b010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095eba8c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f1c5110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b720b930>, 2329805.839381766)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e6c98d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b627b1c0>, 2329804.843119915)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ee42c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5fb8210>, 2329804.918492622)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ee29090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee730130>, 2329804.815257124)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f08fdd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5fbb540>, 2329804.790910614)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f40ae50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5df6b30>, 2329803.80791171)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ed7ae50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b68ddbe0>, 2329805.018142906)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ed436d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b68dc590>, 2329804.954788694)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095eed5290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b68dfe00>, 2329805.13393508)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095eb6a290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5df47c0>, 2329805.014136015)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095eba8890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e216350>, 2329805.555452812)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ead2150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095eaac790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b756db00>, 2329805.82499464)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095eaada50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e7b8290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee7f1550>, 2329805.587933096)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e7b9cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e7c1f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6b5ba10>, 2329805.469158285)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e7c22d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e8149d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e6954d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e449d30>, 2329805.917227052)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e815250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6b58fa0>, 2329805.819541516)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ed42b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e4d4050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67f8c0>, 2329805.75542978)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f60c510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ec25c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f797310>, 2329805.910878234)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e82bf10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e560410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e4aa9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e44ac80>, 2329806.026081376)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e5625d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f797a10>, 2329806.119293027)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e4abc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e814350>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e25a750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b756c9f0>, 2329805.986461639)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e815710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6278050>, 2329806.038137991)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095eb0ff10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ed1c390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e556ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e7fa190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e88f2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e3b6910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f796660>, 2329806.53882051)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e372590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7208f30>, 2329804.715524077)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b445cdd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e78b540>, 2329805.440350051)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e7f9750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b746f380>, 2329805.625435433)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e88d410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6b5b7e0>, 2329805.618146254)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e3b52d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b59daad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095c630590>, 2329806.210819641)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b59dbf10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f715490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e2b5d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f7957f0>, 2329806.422905651)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e03d2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5d246d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a15520c20>, 2329806.436919953)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5d269d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ee830d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a15523230>, 2329806.366648099)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e401310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b48a1f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f67f620>, 2329806.483153883)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6eb8950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e199c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a15521400>, 2329806.599907903)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e199710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f52a990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b699bf50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a15523a80>, 2329806.764283413)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f5293d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a155211d0>, 2329806.633779687)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b699b190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f680190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f57a350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b746dd30>, 2329806.907385406)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f5f6a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a155222e0>, 2329806.925677469)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f57b490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e766e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e8a8910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e222fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e78a970>, 2329806.391788298)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e2b7dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e449080>, 2329806.031438889)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e8a8c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f797770>, 2329806.747841742)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e7666d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6b5bf50>, 2329806.447254463)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e223510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ebe8ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee747c40>, 2329805.879275815)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e2fad90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b48d6cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5d66a50>, 2329807.004658405)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b48d6690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f87ef90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4676dd0>, 2329813.118843211)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f87fc50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f6673d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f850090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b6279ef0>, 2329807.139996386)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f667c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4675e80>, 2329806.959067647)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f8514d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b41b6a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4797ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4674bb0>, 2329807.241496489)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b41b7650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a15523690>, 2329807.416012484)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6176890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f78f710>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b46777e0>, 2329807.230739715)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f78e190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b58ffa10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b41dcd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f7966d0>, 2329807.57762826)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee9dbfd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b68df690>, 2329807.149181263)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b41dff50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6937650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4677850>, 2329807.476321701)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5be8e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f99a1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a15521710>, 2329807.334507137)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f815f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b442c890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e063490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f9a4810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e2b73d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a15520600>, 2329806.744151585)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e060710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5d66970>, 2329806.805888037)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e2b4f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5d679a0>, 2329806.892319149)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b882cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b46748a0>, 2329807.50464913)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b442c090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b76f27d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b583fe70>, 2329807.634512963)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b76f1f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b412e110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f919b70>, 2329807.48352417)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f9d5c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5f78710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4eca9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a15523b60>, 2329807.978669989)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5f7bb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f9197f0>, 2329808.028308189)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4ecab50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b7374fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4420c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f9183d0>, 2329807.781829102)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b73747d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f91a7b0>, 2329807.813771197)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4421350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f851f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4ef9290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f919cc0>, 2329808.057602759)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f7c5b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a15522970>, 2329808.129640099)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4ef9710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5d790d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f91aac0>, 2329808.094642181)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150b15fdbf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b599d050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095c630c90>, 2329808.120139093)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5791790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f730290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b572e150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f8b9390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4906790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b743bf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b74ee710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f91b700>, 2329807.444312896)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e2a8e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f918830>, 2329807.659190557)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b49040d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f918a60>, 2329807.650746964)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b515add0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ed3f8c0>, 2329808.062460199)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b76c5c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a15520830>, 2329808.289198384)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b572c450>\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b46771c0>, 2329807.794672187)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b74edd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150a15409c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150a15520fa0>, 2329808.357814012)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a1540a350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee610750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5adcde0>, 2329808.397359773)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee610d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4816b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee86a9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5adedd0>, 2329808.557065876)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b60130d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b76eb690>, 2329808.434621452)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee8691d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b56df9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee939e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5adc830>, 2329808.827776225)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee938d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4420dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5b77610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4675550>, 2329808.57773404)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4422390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ef96430>, 2329808.765581252)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5b77350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b61e4950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f918590>, 2329808.932015451)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b61e4190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b49aabd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ebf75b0>, 2329808.963094429)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b49ab090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5443090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4802c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b47ffcd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b55454d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b72aca50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f918c90>, 2329808.512700586)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b56dfd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5adf070>, 2329808.29705682)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b44d8690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee7f0600>, 2329808.361871562)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4a7a950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5add160>, 2329808.886160868)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4960510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f91a120>, 2329808.45211433)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a1542eb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f91bc40>, 2329808.521421605)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b72ac6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6ed8bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f91a0b0>, 2329809.041695144)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150b3c0f7e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b61657d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b598ecd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b71150f0>, 2329809.319924914)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ef26990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4677d20>, 2329809.42949279)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b598eb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee8557d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7117cb0>, 2329809.236322955)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee854fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee868490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b470b250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ebf4f30>, 2329809.178135589)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee868b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b71172a0>, 2329809.187766691)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4708c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b69225d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7115be0>, 2329809.539377579)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6922090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b754f8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b608c990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b75ac310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5adf1c0>, 2329809.385145304)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6faad50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eae2cf0>, 2329809.587440005)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b69a2650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b49cadd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b626c0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b55361d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b59fd150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b46765f0>, 2329809.428283308)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a154ac1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5add400>, 2329808.863004941)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b626c2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ebf5240>, 2329808.982952569)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5535110>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6ecbed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eae3ee0>, 2329809.701720985)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4a59050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7114670>, 2329809.171275658)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a1503bbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7114bb0>, 2329809.131641381)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6ec89d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095eb67450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eae0de0>, 2329809.631497841)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095eb67f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095edc6a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e9ba510>, 2329809.830914172)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095edc78d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ee6eb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e9bbe70>, 2329812.892068349)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e1480d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e39f950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ebf5a20>, 2329810.056677058)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e39c850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b74dc490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e57a9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e9b8440>, 2329810.402982986)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b74dc110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e9b9da0>, 2329810.221252727)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e466bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ee0fc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e9ba890>, 2329809.928020301)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ee0ec50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c567190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e9ba430>, 2329810.211311741)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c566c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5d45150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e8afd90>, 2329810.381381015)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e03b110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee662250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e8ad7f0>, 2329810.330661381)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b579aad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b4dd8610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b74811d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509ee930950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095e039990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ef6a0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f406890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eae3bd0>, 2329809.636696635)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e0388d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5add780>, 2329809.602501897)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee932d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b76e9320>, 2329809.801080014)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6ec2d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e8ae190>, 2329810.340305189)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b476a5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e8adb00>, 2329810.658509502)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ef6a110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e9baa50>, 2329810.066395022)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095f407a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c601650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e9bb230>, 2329810.664146758)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5864350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c6f4050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094a88f690>, 2329810.694252608)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c6f5a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b62293d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ef947c0>, 2329810.655090616)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b5e5d810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c55bc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c797090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095eae2510>, 2329811.132757474)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c55f590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094a88db00>, 2329810.909799775)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c55c810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c545150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094aba3c40>, 2329811.228298071)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c545b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c6e05d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094ac9b930>, 2329811.19350236)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c6e0b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c5240d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c767150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c5305d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c641bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c54a190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c6b01d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c622e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b49b4c50>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e8afc40>, 2329810.681564339)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b489dc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e8af380>, 2329810.468986079)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c621290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e8ac4b0>, 2329810.529327494)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c55d5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e9bb3f0>, 2329810.647426859)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c533a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094ac9baf0>, 2329811.089028775)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c524690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094a88f850>, 2329811.226870004)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c76d4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e8ae350>, 2329810.275407705)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c640c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5ade040>, 2329810.603188146)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c549190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c78cf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c88c7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e684f30>, 2329811.35206138)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c7d1ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094a88c9f0>, 2329811.665410557)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c88cd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c71d7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c9cd7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e685400>, 2329811.528124902)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c71fed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b446ec10>, 2329811.437679975)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c9cd450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f10c990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b446c3d0>, 2329811.869545866)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c964810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c8d24d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c7364d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c8f8d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c8c9a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095f8ba4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c71c910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c9f6890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c8e2990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094a88f770>, 2329811.267458617)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c735810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094a88df60>, 2329811.309881463)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509ee662c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e685240>, 2329811.521904803)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c8db6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b446d7f0>, 2329812.148182555)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c8d0310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c885c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e8afe00>, 2329811.480548583)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b53eff50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094a88f310>, 2329811.526883722)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c723750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b446dbe0>, 2329811.568563572)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c9f6a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e684c90>, 2329811.236914351)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c8e3e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e8ae580>, 2329811.387470296)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e6634d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ca84310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15094aba2580>, 2329812.270282945)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ca85190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c936650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e2eba10>, 2329812.194728014)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c936710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cbb2c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e2e80c0>, 2329812.29484311)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cbb05d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cab5250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cb69950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e2ebd90>, 2329812.222907918)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cab6dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cb44310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e684520>, 2329812.244976456)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cb46cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cacc090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cb23010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e2e9630>, 2329812.367376039)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cacc2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e2e8f30>, 2329812.481545762)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cb22650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ccea690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ccc6e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e2ea5f0>, 2329812.421257921)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ccea750>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c9f5010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c96c7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c7d24d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ca1f2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cbed290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e9ba660>, 2329812.202761827)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cb68790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f9edcc0>, 2329813.089026386)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ccc6ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e2e9e80>, 2329812.035072441)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c97dad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b446e820>, 2329811.969659137)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095e28b510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e2eba80>, 2329812.046682651)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c96c550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cbd9510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b446ce50>, 2329811.838168053)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c9f5350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e2eb2a0>, 2329812.143864801)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b49b4950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095ed3eac0>, 2329812.053135135)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cbd9e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6686210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e685010>, 2329812.689407078)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c8d2990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cc102d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cded590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f9ed860>, 2329812.826086732)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cc10510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f9ef460>, 2329813.37531048)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cdef150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cdc5e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cc40110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453c0c0>, 2329814.229922175)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cdc7650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e686190>, 2329812.978003638)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cc5f150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cce9ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cd132d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453fa10>, 2329812.888080277)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c88d4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453d4e0>, 2329812.872521673)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cd10710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cd06a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453c440>, 2329813.03262425)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cd06c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cef2510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cebffd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e2e82f0>, 2329813.015837886)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cef27d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f9ee510>, 2329813.019703232)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cebe310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cdd7790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ce9c550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453c9f0>, 2329813.151819412)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cee4490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453e7b0>, 2329813.291075215)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ce7c450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ce70ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453ff50>, 2329813.112295654)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ce73350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ce59650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453d320>, 2329813.199585672)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ce59d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cb20a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cdaa110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095c524d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b446f620>, 2329812.782563908)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cc17690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453d7f0>, 2329813.313455222)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095c890d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e2ea660>, 2329812.856971568)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b4619310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cf44c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f9ef540>, 2329813.610239372)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cf44b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cf379d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e2eac10>, 2329813.708890154)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cf36890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cfdae10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cf12150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e8aef90>, 2329813.790350446)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ce636d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4f95780>, 2329813.690994544)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cf10650>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cfc49d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d0ac2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4f95630>, 2329813.784130312)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cf36710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7109be0>, 2329813.873504604)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cfbc610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d056450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b710b380>, 2329813.844064893)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d055f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d025f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7109e10>, 2329813.950618179)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d027150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cebcb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d007c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b710bcb0>, 2329813.900913029)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cebc210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b710b460>, 2329813.773958244)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d007c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cf91710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d1b8cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7109fd0>, 2329813.864983954)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cf90150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b710aa50>, 2329814.106747131)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d1bb950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d1c2190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d191890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453ef90>, 2329813.918290059)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d1c3010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7109a20>, 2329814.031940566)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d191110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d17b010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095ccf47d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b7109b00>, 2329813.990948243)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d178550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e2ebee0>, 2329813.603477029)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095ccf43d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d1483d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d2faa90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee929cc0>, 2329814.115049793)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d148d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee9292b0>, 2329814.179348141)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d1054d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d282e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee928670>, 2329814.443387458)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d283b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cfa6c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee92b1c0>, 2329814.345773756)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d2b2e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d1aab10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d21a310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee928280>, 2329814.553635174)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d21b410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d2d72d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d9f5f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4f94d00>, 2329814.502089725)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d2d48d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4f95cc0>, 2329814.666588512)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d9f7990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d230050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d384810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e55b4d0>, 2329814.558344304)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d231cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e558360>, 2329814.794514671)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d385590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d392e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d3054d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4f946e0>, 2329814.592785454)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cfe6550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e558de0>, 2329814.661037141)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d306410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d34d9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d150610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee929be0>, 2329814.458580801)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d1aa690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e5596a0>, 2329815.723940618)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d34c690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4676040>, 2329814.209637171)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d1526d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cf11590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee928ad0>, 2329815.075150254)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cf125d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d298b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee92b9a0>, 2329814.727809918)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d298bd0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d22e150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d489a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e32e900>, 2329861.678986951)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d22fdd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e32d940>, 2329815.065218889)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d488290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d45c490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e32e0b0>, 2329862.07234516)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d45c290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d44b2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4f94980>, 2329814.940978933)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d44b390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d2bf4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e32da20>, 2329815.158116568)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d2bf950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d4ef650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b453cad0>, 2329815.088832689)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d4efd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d5dbd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e32fd20>, 2329815.391416603)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d8a5f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d3e7510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d6ad610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e32ed60>, 2329815.3509594)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d3e5c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e32f230>, 2329815.657395009)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d6afd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095cf46150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d560750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d318790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d4bd490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e55a9e0>, 2329814.673043412)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d31a250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b446fee0>, 2329814.703524464)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d3f4c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f43cd00>, 2329815.320661983)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d247b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e2e94e0>, 2329863.4107566)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d562190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d4e81d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f43ee40>, 2329815.448872767)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d4ebd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d692010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5eda4a0>, 2329862.083359572)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d691d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d75fa50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d668b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5ed83d0>, 2329816.014637006)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d668b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5d33450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5e24520>, 2329863.3226731)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095cf44dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d93c210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5e26660>, 2329862.936351484)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d93d6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d8754d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e32cde0>, 2329863.163570952)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d77b5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d5182d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d889fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d54e3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d531610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d917d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d7a2950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e55a0b0>, 2329862.148874874)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d75dbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b6b0dc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4f96430>, 2329815.45309005)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509b6b0d0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f43c830>, 2329815.635614534)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d7a3310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d67ed10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f43df60>, 2329816.399459972)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d533d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f43cf30>, 2329816.041286021)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d67cf10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e32f620>, 2329863.367414773)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d7e7e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5e25d30>, 2329862.185111667)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d889ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f43f7e0>, 2329815.385046692)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d54ebd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f43ce50>, 2329863.239411833)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d5036d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d8e9250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5e24d70>, 2329816.235625159)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150a153ca150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d85dc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5e27e70>, 2329863.278973654)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d85d690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d4b2a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b49500c0>, 2329862.772255407)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d4b1dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150949e5eed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4953930>, 2329863.05962928)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150949e86e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150949ea7690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4952e40>, 2329865.098868823)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150949ea7550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150949c0f090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d819a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094bc89d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15095d4bfc50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094a675a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150949e7f250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee68d9b0>, 2329863.526583915)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094bc89790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4950e50>, 2329863.558384735)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b827a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150949decbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5e25b70>, 2329862.900691592)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150949c0dc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5e255c0>, 2329863.200303623)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d540bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4952350>, 2329863.67116278)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094a676f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095f43f850>, 2329862.82064165)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150949e7f050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150949dd7000>, 2329863.709488092)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150949ded2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150949c5f550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150949c3cd00>, 2329863.848469257)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d7bf590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150949c80ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e32f850>, 2329863.831371988)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150949c7b6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150949c948d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150949cb1a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150949c3e6d0>, 2329863.986735407)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150949cb2110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150949c65610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150949ccd510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150949c3cbb0>, 2329865.046509392)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150949c64d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150949c3ed60>, 2329864.068988757)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150949ccdc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150949c66a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150949cea8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b63c7460>, 2329864.078660057)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150949c670d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150949c3f540>, 2329865.005541996)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150949cea6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150949cfd610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150949c3faf0>, 2329865.034508839)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150949cfd450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150949b1a810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150949e78ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509b5a5d850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150949c3c130>, 2329863.967400279)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150949c946d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee68dbe0>, 2329865.785437597)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15095d97bf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094baad190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee68c830>, 2329864.015467185)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094baac150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150949c5dc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150949dd7e70>, 2329865.384406484)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150949b18950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095e32dda0>, 2329863.000767325)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150949e79bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150949dd7a10>, 2329863.639485624)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150949c5d850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150949c39d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15095c95b4d0>, 2329863.645460392)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150949c3b5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150949b22950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150949b29be0>, 2329865.823464203)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150949b9a690>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150949ce4d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150949b2a820>, 2329865.864360898)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150949ca82d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150949b7ed50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150949d5c210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150949ccf6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150949b524d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150949cb23d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150949b73850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150949bd7390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150949be3850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150949a18050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150949b53110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150949a47c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150949a3cd70>, 2329865.903860038)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150949b7ef50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150949b70dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5e240c0>, 2329865.2281743)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150949d5c2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b5eda270>, 2329865.40830421)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094a6436d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150949c3d400>, 2329865.282478941)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150949b51890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150949c3d940>, 2329865.627192531)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150949cb1d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150949c3f380>, 2329865.523918668)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150949cca4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee68fcb0>, 2329865.756293193)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094b88c390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150949b2b690>, 2329865.742711627)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150949be3750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509b4950050>, 2329865.77599422)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150949a183d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150949b2bee0>, 2329865.800716529)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150949b51790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150949a3cad0>, 2329866.001979502)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150949a47b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150949c3d240>, 2329865.504341642)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150949b70a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150949ad4f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150949b2ae40>, 2329866.021158907)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150949ad4e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150949905390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150949a3f230>, 2329866.13587231)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150949905c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094990f410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1509ee9289f0>, 2329866.170742373)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094990fb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150949945c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150949a825d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150949ac5f60>, 2329866.42688401)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150949a82250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150949973910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150949ac65f0>, 2329867.315180698)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150949973c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150949980190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150949dd6900>, 2329866.695611552)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150949980a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15094999cf10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1509499ab910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150949a81e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150949a8a6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150949a9bc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150949ab1b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x150949ae5390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150949ac5d30>, 2329866.275466858)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150949945a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150949a3f850>, 2329865.899640461)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150949ab2610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150949a3ec10>, 2329865.862225086)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150949cca710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150949ac4980>, 2329866.011409934)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150949ae5190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150949a3ef20>, 2329866.527875076)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15094999cd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150949ac70e0>, 2329866.970563121)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1509499abe10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150949a3e4a0>, 2329865.754360194)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150949a82550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x150949a3f070>, 2329865.718658148)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x150949a9bb50>\n",
      "Evaluating workflow:  80%|███████▉  | 2387/3000 [08:16<00:47, 13.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 2393/3000 [08:16<00:33, 17.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 2396/3000 [08:16<00:36, 16.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2400/3000 [08:17<00:39, 15.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 2402/3000 [09:02<38:23,  3.85s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 2405/3000 [09:03<26:38,  2.69s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2407/3000 [09:03<20:04,  2.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 2411/3000 [09:03<11:16,  1.15s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2413/3000 [09:03<08:20,  1.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2418/3000 [09:04<04:09,  2.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2427/3000 [09:04<01:36,  5.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2432/3000 [09:05<01:35,  5.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2436/3000 [09:05<01:20,  7.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████▏ | 2440/3000 [09:06<00:54, 10.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2449/3000 [09:06<00:30, 18.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2452/3000 [09:06<00:32, 16.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2461/3000 [09:06<00:25, 21.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2469/3000 [09:07<00:20, 26.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2477/3000 [09:07<00:26, 19.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2486/3000 [09:08<00:19, 26.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2490/3000 [09:08<00:23, 21.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2496/3000 [09:08<00:22, 22.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2500/3000 [09:08<00:19, 26.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▎ | 2507/3000 [09:09<00:21, 23.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2515/3000 [09:09<00:18, 25.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2518/3000 [09:09<00:21, 22.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2522/3000 [09:09<00:22, 21.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2532/3000 [09:10<00:21, 21.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2536/3000 [09:10<00:21, 22.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▍ | 2545/3000 [09:10<00:16, 27.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2549/3000 [09:10<00:17, 25.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2553/3000 [09:11<00:18, 24.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2556/3000 [09:11<00:22, 19.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▌ | 2562/3000 [09:11<00:22, 19.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2571/3000 [09:11<00:15, 27.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2575/3000 [09:12<00:21, 19.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2584/3000 [09:12<00:16, 25.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▋ | 2588/3000 [09:12<00:18, 22.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▋ | 2591/3000 [09:12<00:22, 18.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      " {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2597/3000 [09:13<00:19, 20.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2603/3000 [09:13<00:18, 21.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2606/3000 [09:13<00:20, 19.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2613/3000 [09:13<00:16, 23.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2616/3000 [09:14<00:22, 17.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2619/3000 [09:14<00:22, 17.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2626/3000 [09:14<00:17, 21.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2629/3000 [09:14<00:21, 16.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2636/3000 [09:15<00:13, 26.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2640/3000 [09:15<00:17, 20.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2644/3000 [09:15<00:15, 23.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2648/3000 [09:15<00:16, 21.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2654/3000 [09:16<00:18, 19.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▊ | 2657/3000 [09:16<00:35,  9.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▊ | 2659/3000 [09:17<00:36,  9.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2666/3000 [09:17<00:24, 13.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2669/3000 [09:18<00:41,  8.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2670/3000 [10:03<29:37,  5.39s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 10:03:40.961\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in gene expression analysis. For K562 cells, assess whether perturbation of SCYL1 is associated with a significant change in FCGRT expression compared with unperturbed controls. Answer strictly as 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are an expert in gene expression analysis. For K562 cells, assess whether perturbation of SCYL1 is associated with a significant change in FCGRT expression compared with unperturbed controls. Answer strictly as 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 3535}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2674/3000 [10:03<17:39,  3.25s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2679/3000 [10:03<10:13,  1.91s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|████████▉ | 2685/3000 [10:04<05:32,  1.06s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 2688/3000 [10:04<04:02,  1.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|████████▉ | 2695/3000 [10:04<01:59,  2.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 2698/3000 [10:05<01:55,  2.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2700/3000 [10:06<01:39,  3.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|█████████ | 2709/3000 [10:06<00:40,  7.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|█████████ | 2715/3000 [10:06<00:29,  9.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2722/3000 [10:07<00:19, 14.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2730/3000 [10:07<00:13, 20.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2733/3000 [10:07<00:12, 21.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████▏| 2739/3000 [10:07<00:12, 20.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2745/3000 [10:08<00:12, 19.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2751/3000 [10:08<00:09, 25.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2754/3000 [10:08<00:11, 20.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2757/3000 [10:08<00:12, 19.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2764/3000 [10:08<00:11, 20.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2771/3000 [10:09<00:10, 22.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2774/3000 [10:09<00:12, 18.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2778/3000 [10:09<00:09, 22.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2784/3000 [10:09<00:10, 21.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2787/3000 [10:09<00:09, 22.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2796/3000 [10:10<00:08, 22.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2803/3000 [10:10<00:08, 23.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▎| 2811/3000 [10:10<00:06, 29.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2815/3000 [10:11<00:09, 19.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2822/3000 [10:11<00:07, 23.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2825/3000 [10:11<00:07, 22.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2830/3000 [10:11<00:06, 25.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▍| 2838/3000 [10:12<00:06, 25.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 2842/3000 [10:12<00:07, 20.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 2845/3000 [10:12<00:07, 21.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▌| 2851/3000 [10:12<00:07, 20.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▌| 2859/3000 [10:13<00:05, 23.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2862/3000 [10:13<00:05, 23.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2868/3000 [10:13<00:06, 19.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2872/3000 [10:13<00:05, 22.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2878/3000 [10:14<00:05, 22.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2884/3000 [10:14<00:05, 23.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2887/3000 [10:14<00:05, 20.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▋| 2890/3000 [10:14<00:05, 20.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2898/3000 [10:15<00:04, 22.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2902/3000 [10:15<00:03, 24.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2905/3000 [10:15<00:05, 18.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2912/3000 [10:15<00:03, 22.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2915/3000 [10:16<00:05, 14.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2918/3000 [10:16<00:04, 16.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2921/3000 [10:16<00:06, 12.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2926/3000 [10:17<00:07,  9.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2931/3000 [10:17<00:05, 11.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2933/3000 [10:17<00:06, 10.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2026-01-13 10:03:55.977\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in gene expression analysis. For K562 cells, assess whether perturbation of YIPF5 is associated with a significant change in IL32 expression compared with unperturbed controls. Answer strictly as 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are an expert in gene expression analysis. For K562 cells, assess whether perturbation of YIPF5 is associated with a significant change in IL32 expression compared with unperturbed controls. Answer strictly as 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 565}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 45 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 10:03:55.977\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are assisting with the interpretation of perturbation-based expression data. In K562 cells, SAMM50 is perturbed and NUF2 expression is quantified. Does this perturbation result in a significant change in NUF2 expression compared with control cells? Respond exactly as 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are assisting with the interpretation of perturbation-based expression data. In K562 cells, SAMM50 is perturbed and NUF2 expression is quantified. Does this perturbation result in a significant change in NUF2 expression compared with control cells? Respond exactly as 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 1392}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 45 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2941/3000 [11:04<03:29,  3.55s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2952/3000 [11:04<00:55,  1.15s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▊| 2956/3000 [11:04<00:37,  1.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2965/3000 [11:05<00:14,  2.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2969/3000 [11:06<00:11,  2.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2972/3000 [11:06<00:08,  3.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2975/3000 [11:06<00:06,  4.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2981/3000 [11:06<00:02,  6.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|█████████▉| 2989/3000 [11:07<00:00, 11.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2992/3000 [11:07<00:00, 11.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|█████████▉| 2998/3000 [11:07<00:00, 14.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|██████████| 3000/3000 [11:09<00:00,  4.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 10:04:46.514\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m<module>\u001b[0m:\u001b[36m4\u001b[0m - \u001b[1mEvaluation metrics (before optimization): {'f1': 0.0, 'em': 0.0, 'acc': 0.5223631508678238}\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "logger.info(\"Evaluating workflow on test set...\")\n",
    "with suppress_logger_info():\n",
    "    results = textgrad_optimizer.evaluate(dataset=benchmark, eval_mode=\"test\")\n",
    "logger.info(f\"Evaluation metrics (before optimization): {results}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "1726fb5f",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'f1': 0.0, 'em': 0.0, 'acc': 0.5223631508678238}"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2042bc81",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "52835725",
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ[\"AZURE_OPENAI_DEPLOYMENT_NAME\"] = \"gpt-4o-mini\"\n",
    "os.environ[\"AZURE_OPENAI_ENDPOINT\"] = \"https://tianyuliu-hua-raredisea-resource.cognitiveservices.azure.com/\"\n",
    "os.environ[\"AZURE_OPENAI_KEY\"] = \"2pa9h2ZIN1lQepFWwYADlXIKIansa9KPhxMoumeGbRQ08f2uDTXiJQQJ99BKACHYHv6XJ3w3AAAAACOGsQIt\"\n",
    "os.environ[\"AZURE_OPENAI_API_VERSION\"] = \"2025-01-01-preview\"\n",
    "llm_config = LiteLLMConfig(model=\"azure/\" + os.getenv(\"AZURE_OPENAI_DEPLOYMENT_NAME\"),  # Azure model format\n",
    "    azure_endpoint=os.getenv(\"AZURE_OPENAI_ENDPOINT\"),\n",
    "    azure_key=os.getenv(\"AZURE_OPENAI_KEY\"),\n",
    "    api_version=os.getenv(\"AZURE_OPENAI_API_VERSION\", \"2024-12-01-preview\"), top_p=0.85, temperature=0.2, frequency_penalty=0.0, presence_penalty=0.0)\n",
    "\n",
    "executor_llm = LiteLLM(config=llm_config)\n",
    "optimizer_llm = LiteLLM(config=llm_config)\n",
    "llm = executor_llm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4e2e88c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "hotpotqa_graph_data = {\n",
    "    \"goal\": \"Answer the question based on the context. The answer should be a direct response to the question, without including explanations or reasoning.\",\n",
    "    \"tasks\": [\n",
    "        {\n",
    "            \"name\": \"answer_generate\",\n",
    "            \"description\": \"Answer the question based on the context.\",\n",
    "            \"inputs\": [\n",
    "                {\"name\": \"problem\", \"type\": \"str\", \"required\": True, \"description\": \"The problem to solve.\"}\n",
    "            ],\n",
    "            \"outputs\": [\n",
    "                {\"name\": \"answer\", \"type\": \"str\", \"required\": True, \"description\": \"The answer to the problem.\"}\n",
    "            ],\n",
    "            \"prompt_template\": StringTemplate(instruction=\"Think step by step to answer the question. You should explain your thinking process in the 'thought' field, and provide the final answer in the 'answer' field.\\nFormat your output in xml format, such as <thought>xxx</thought> and <answer>xxx</answer>.\"),\n",
    "            \"parse_mode\": \"xml\"\n",
    "        }\n",
    "    ] \n",
    "}\n",
    "\n",
    "def collate_func(example: dict) -> dict:\n",
    "    problem = \"Question: {}\\n\\nAnswer:\".format(example[\"question_new\"])\n",
    "    return {\"problem\": problem}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "297c05c2",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:41:26.472\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.benchmark.pertqa\u001b[0m:\u001b[36m_load_data_from_file\u001b[0m:\u001b[36m52\u001b[0m - \u001b[1mloading HotPotQA data from /home/tl688/pitl688/selfevolve/EvoAgentX/examples/pertqa/adamson_update_train.json ...\u001b[0m\n",
      "\u001b[32m2026-01-13 12:41:26.476\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.benchmark.pertqa\u001b[0m:\u001b[36m_load_data_from_file\u001b[0m:\u001b[36m52\u001b[0m - \u001b[1mloading HotPotQA data from /home/tl688/pitl688/selfevolve/EvoAgentX/examples/pertqa/adamson_update_train.json ...\u001b[0m\n",
      "\u001b[32m2026-01-13 12:41:26.479\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.benchmark.pertqa\u001b[0m:\u001b[36m_load_data_from_file\u001b[0m:\u001b[36m52\u001b[0m - \u001b[1mloading HotPotQA data from /home/tl688/pitl688/selfevolve/EvoAgentX/examples/pertqa/adamson_update_test.json ...\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "# llm_config = OpenAILLMConfig(model=\"gpt-4o-mini-2024-07-18\", openai_key=OPENAI_API_KEY, top_p=0.85, temperature=0.2, frequency_penalty=0.0, presence_penalty=0.0)\n",
    "# llm = OpenAILLM(config=llm_config)\n",
    "llm = executor_llm\n",
    "\n",
    "# obtain SEW workflow \n",
    "sew_graph = SEWWorkFlowGraph.from_dict(hotpotqa_graph_data)\n",
    "agent_manager = AgentManager()\n",
    "agent_manager.add_agents_from_workflow(sew_graph, executor_llm.config)\n",
    "\n",
    "benchmark = PertQA(pertdata='adamson')\n",
    "\n",
    "# obtain Evaluator\n",
    "evaluator = Evaluator(llm=llm, agent_manager=agent_manager, collate_func=collate_func, num_workers=20, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "3c048529",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# obtain SEWOptimizer after having more roles, default\n",
    "optimizer = SEWOptimizer(\n",
    "    graph=sew_graph, \n",
    "    evaluator=evaluator, \n",
    "    llm=llm, \n",
    "    max_steps=20,\n",
    "    eval_rounds=3, \n",
    "    repr_scheme=\"python\", \n",
    "    optimize_mode=\"all\", \n",
    "    order=\"zero-order\",\n",
    "    max_rounds=20,\n",
    ")\n",
    "\n",
    "# with suppress_logger_info():\n",
    "#     metrics = optimizer.evaluate(dataset=humaneval, eval_mode=\"test\")\n",
    "# print(\"Evaluation metrics: \", metrics)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "9cbdd001",
   "metadata": {},
   "outputs": [],
   "source": [
    "# len(benchmark._train_data)\n",
    "import numpy as np\n",
    "np.random.seed(2024)\n",
    "out = np.random.choice(benchmark._train_data, size=150, replace=False)\n",
    "benchmark._train_data = out\n",
    "benchmark._dev_data = out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8b05058e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:41:26.506\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m678\u001b[0m - \u001b[1mOptimizing the SEWWorkFlowGraph workflow with python representation.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:41:26.506\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m682\u001b[0m - \u001b[1mRun initial evaluation on the original workflow ...\u001b[0m\n",
      "Evaluating workflow:   1%|          | 1/150 [00:02<06:03,  2.44s/it]Task exception was never retrieved\n",
      "future: <Task finished name='Task-131' coro=<_client_async_logging_helper() done, defined at /gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/site-packages/litellm/utils.py:855> exception=RuntimeError('Event loop is closed')>\n",
      "Traceback (most recent call last):\n",
      "  File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/asyncio/tasks.py\", line 277, in __step\n",
      "    result = coro.send(None)\n",
      "             ^^^^^^^^^^^^^^^\n",
      "  File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/site-packages/litellm/utils.py\", line 873, in _client_async_logging_helper\n",
      "    GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(\n",
      "  File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/site-packages/litellm/litellm_core_utils/logging_worker.py\", line 322, in ensure_initialized_and_enqueue\n",
      "    self.enqueue(async_coroutine)\n",
      "  File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/site-packages/litellm/litellm_core_utils/logging_worker.py\", line 131, in enqueue\n",
      "    self._queue.put_nowait(task)\n",
      "  File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/asyncio/queues.py\", line 147, in put_nowait\n",
      "    self._wakeup_next(self._getters)\n",
      "  File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/asyncio/queues.py\", line 63, in _wakeup_next\n",
      "    waiter.set_result(None)\n",
      "  File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/asyncio/futures.py\", line 263, in set_result\n",
      "    self.__schedule_callbacks()\n",
      "  File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/asyncio/futures.py\", line 173, in __schedule_callbacks\n",
      "    self._loop.call_soon(callback, self, context=ctx)\n",
      "  File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/asyncio/base_events.py\", line 762, in call_soon\n",
      "    self._check_closed()\n",
      "  File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/asyncio/base_events.py\", line 520, in _check_closed\n",
      "    raise RuntimeError('Event loop is closed')\n",
      "RuntimeError: Event loop is closed\n",
      "Evaluating workflow:   2%|▏         | 3/150 [00:02<01:42,  1.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 6/150 [00:02<00:45,  3.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 11/150 [00:02<00:20,  6.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 15/150 [00:03<00:13, 10.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 18/150 [00:03<00:11, 11.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 21/150 [00:04<00:20,  6.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 23/150 [00:04<00:22,  5.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 27/150 [00:05<00:17,  6.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 30/150 [00:05<00:12,  9.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 34/150 [00:05<00:11, 10.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 36/150 [00:05<00:11,  9.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 38/150 [00:06<00:13,  8.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 40/150 [00:06<00:11,  9.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 42/150 [00:06<00:14,  7.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███▏      | 47/150 [00:07<00:12,  8.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 49/150 [00:07<00:11,  8.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 54/150 [00:08<00:10,  9.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 56/150 [00:08<00:11,  7.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▊      | 58/150 [00:08<00:12,  7.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████▏     | 62/150 [00:09<00:10,  8.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 64/150 [00:09<00:12,  6.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 65/150 [00:09<00:13,  6.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 70/150 [00:10<00:08,  9.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 72/150 [00:10<00:06, 11.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 74/150 [00:10<00:06, 11.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 76/150 [00:11<00:10,  7.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 80/150 [00:11<00:09,  7.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 83/150 [00:12<00:09,  7.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 87/150 [00:12<00:08,  7.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|██████    | 90/150 [00:12<00:07,  7.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 93/150 [00:13<00:06,  8.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 95/150 [00:13<00:05, 10.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 97/150 [00:13<00:06,  8.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 99/150 [00:13<00:06,  8.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 100/150 [00:14<00:09,  5.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 104/150 [00:14<00:05,  8.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 106/150 [00:15<00:06,  6.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 111/150 [00:15<00:03,  9.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 114/150 [00:15<00:03,  9.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 120/150 [00:16<00:03,  9.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 123/150 [00:16<00:03,  8.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 126/150 [00:17<00:03,  7.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 128/150 [00:17<00:02,  9.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▊ | 133/150 [00:17<00:01, 10.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████▏| 137/150 [00:18<00:01, 11.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 139/150 [00:18<00:01, 10.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 141/150 [00:18<00:01,  7.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 145/150 [00:19<00:00,  9.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 147/150 [00:19<00:00,  8.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▊| 148/150 [00:19<00:00,  5.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|██████████| 150/150 [00:20<00:00,  7.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:41:47.045\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m685\u001b[0m - \u001b[1mInitial metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.3933333333333333}\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:41:54.077\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.031 | Total tokens: 118718 | Current cost: $0.000 | Current tokens: 551\u001b[0m\n",
      "\u001b[32m2026-01-13 12:41:54.945\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.032 | Total tokens: 119318 | Current cost: $0.000 | Current tokens: 600\u001b[0m\n",
      "\u001b[32m2026-01-13 12:41:54.947\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 0: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:41:56.029\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.032 | Total tokens: 119421 | Current cost: $0.000 | Current tokens: 103\u001b[0m\n",
      "\u001b[32m2026-01-13 12:41:57.175\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.032 | Total tokens: 119629 | Current cost: $0.000 | Current tokens: 208\u001b[0m\n",
      "\u001b[32m2026-01-13 12:41:57.176\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36mparse_workflow_python_repr\u001b[0m:\u001b[36m403\u001b[0m - \u001b[33m\u001b[1mFailed to parse workflow string: 'llm_config'. Return the original workflow.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:41:57.176\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 1: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:41:58.166\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.032 | Total tokens: 119740 | Current cost: $0.000 | Current tokens: 111\u001b[0m\n",
      "\u001b[32m2026-01-13 12:41:59.016\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.032 | Total tokens: 119919 | Current cost: $0.000 | Current tokens: 179\u001b[0m\n",
      "\u001b[32m2026-01-13 12:41:59.017\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 2: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:41:59.686\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.032 | Total tokens: 120034 | Current cost: $0.000 | Current tokens: 115\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:00.502\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.032 | Total tokens: 120179 | Current cost: $0.000 | Current tokens: 145\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:00.503\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36mparse_workflow_python_repr\u001b[0m:\u001b[36m403\u001b[0m - \u001b[33m\u001b[1mFailed to parse workflow string: 'llm_config'. Return the original workflow.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:00.503\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 3: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:01.314\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.032 | Total tokens: 120276 | Current cost: $0.000 | Current tokens: 97\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:02.084\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.032 | Total tokens: 120436 | Current cost: $0.000 | Current tokens: 160\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:02.086\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 4: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:04.602\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.032 | Total tokens: 120701 | Current cost: $0.000 | Current tokens: 265\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:07.002\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.032 | Total tokens: 121112 | Current cost: $0.000 | Current tokens: 411\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:07.003\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36mparse_workflow_python_repr\u001b[0m:\u001b[36m403\u001b[0m - \u001b[33m\u001b[1mFailed to parse workflow string: 'llm_config'. Return the original workflow.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:07.003\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 5: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:10.617\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.032 | Total tokens: 121338 | Current cost: $0.000 | Current tokens: 226\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:11.328\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.032 | Total tokens: 121627 | Current cost: $0.000 | Current tokens: 289\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:11.329\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 6: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:12.324\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.032 | Total tokens: 121733 | Current cost: $0.000 | Current tokens: 106\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:13.091\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.032 | Total tokens: 121902 | Current cost: $0.000 | Current tokens: 169\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:13.093\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 7: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:13.851\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.032 | Total tokens: 121981 | Current cost: $0.000 | Current tokens: 79\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:14.635\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.033 | Total tokens: 122133 | Current cost: $0.000 | Current tokens: 152\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:14.636\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36mparse_workflow_python_repr\u001b[0m:\u001b[36m403\u001b[0m - \u001b[33m\u001b[1mFailed to parse workflow string: 'llm_config'. Return the original workflow.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:14.637\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 8: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:15.888\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.033 | Total tokens: 122282 | Current cost: $0.000 | Current tokens: 149\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:17.039\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.033 | Total tokens: 122518 | Current cost: $0.000 | Current tokens: 236\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:17.040\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36mparse_workflow_python_repr\u001b[0m:\u001b[36m403\u001b[0m - \u001b[33m\u001b[1mFailed to parse workflow string: 'llm_config'. Return the original workflow.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:17.040\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 9: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:17.520\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.033 | Total tokens: 122590 | Current cost: $0.000 | Current tokens: 72\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:42:18.199\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.033 | Total tokens: 122717 | Current cost: $0.000 | Current tokens: 127\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:18.201\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 10: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:20.186\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.033 | Total tokens: 122916 | Current cost: $0.000 | Current tokens: 199\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:20.894\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.033 | Total tokens: 123178 | Current cost: $0.000 | Current tokens: 262\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:20.896\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 11: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:21.573\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.033 | Total tokens: 123257 | Current cost: $0.000 | Current tokens: 79\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:22.372\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.033 | Total tokens: 123398 | Current cost: $0.000 | Current tokens: 141\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:22.373\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 12: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:23.141\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.033 | Total tokens: 123477 | Current cost: $0.000 | Current tokens: 79\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:24.000\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.033 | Total tokens: 123628 | Current cost: $0.000 | Current tokens: 151\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:24.001\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36mparse_workflow_python_repr\u001b[0m:\u001b[36m403\u001b[0m - \u001b[33m\u001b[1mFailed to parse workflow string: 'llm_config'. Return the original workflow.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:24.002\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 13: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:24.655\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.033 | Total tokens: 123738 | Current cost: $0.000 | Current tokens: 110\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:25.601\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.033 | Total tokens: 123878 | Current cost: $0.000 | Current tokens: 140\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:25.602\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36mparse_workflow_python_repr\u001b[0m:\u001b[36m403\u001b[0m - \u001b[33m\u001b[1mFailed to parse workflow string: 'llm_config'. Return the original workflow.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:25.603\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 14: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:26.198\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.033 | Total tokens: 123973 | Current cost: $0.000 | Current tokens: 95\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:26.891\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.033 | Total tokens: 124109 | Current cost: $0.000 | Current tokens: 136\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:26.893\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 15: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:27.920\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.033 | Total tokens: 124225 | Current cost: $0.000 | Current tokens: 116\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:28.642\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.033 | Total tokens: 124409 | Current cost: $0.000 | Current tokens: 184\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:28.644\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 16: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:29.766\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.033 | Total tokens: 124517 | Current cost: $0.000 | Current tokens: 108\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:30.519\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.033 | Total tokens: 124693 | Current cost: $0.000 | Current tokens: 176\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:30.520\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 17: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:31.138\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.033 | Total tokens: 124806 | Current cost: $0.000 | Current tokens: 113\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:31.939\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.033 | Total tokens: 124951 | Current cost: $0.000 | Current tokens: 145\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:31.941\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36mparse_workflow_python_repr\u001b[0m:\u001b[36m403\u001b[0m - \u001b[33m\u001b[1mFailed to parse workflow string: 'llm_config'. Return the original workflow.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:31.941\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 18: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:32.985\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.033 | Total tokens: 125074 | Current cost: $0.000 | Current tokens: 123\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:33.772\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.033 | Total tokens: 125265 | Current cost: $0.000 | Current tokens: 191\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:33.773\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 19: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:33.773\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m707\u001b[0m - \u001b[1mReach the maximum number of steps 20. Stop the optimization.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:33.774\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m710\u001b[0m - \u001b[1mRestore the best graph from the snapshot ...\u001b[0m\n",
      "\u001b[32m2026-01-13 12:42:33.774\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36mrestore_best_graph\u001b[0m:\u001b[36m814\u001b[0m - \u001b[1mRestore the best graph from snapshot with metrics {'f1': 0.0, 'em': 0.0, 'acc': 0.3933333333333333} ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   0%|          | 3/3000 [00:01<23:30,  2.12it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   0%|          | 6/3000 [00:01<10:01,  4.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   0%|          | 12/3000 [00:02<04:22, 11.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|          | 18/3000 [00:02<03:12, 15.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|          | 23/3000 [00:03<07:24,  6.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|          | 28/3000 [00:04<05:14,  9.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 30/3000 [00:04<04:55, 10.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|          | 34/3000 [00:04<04:30, 10.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|          | 36/3000 [00:05<05:27,  9.05it/s]Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8125da990>\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|▏         | 38/3000 [00:05<06:20,  7.78it/s]Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8125dea10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81249f070>, 2339451.679323821)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8125db650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8125e9650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c812411290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81249e350>, 2339451.596918764)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8125e9e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81249f1c0>, 2339451.947792122)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c812411a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8124182d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81241a390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81249dc50>, 2339451.710715527)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c812418a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81249f7e0>, 2339452.203190386)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81241aa50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81249eb30>, 2339451.924245545)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8125df010>\n",
      "Evaluating workflow:   1%|▏         | 40/3000 [00:05<08:09,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 46/3000 [00:06<05:47,  8.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 50/3000 [00:07<06:07,  8.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 52/3000 [00:07<05:26,  9.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 55/3000 [00:07<04:31, 10.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 57/3000 [00:08<07:24,  6.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 59/3000 [00:08<07:20,  6.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 63/3000 [00:08<07:00,  6.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 69/3000 [00:09<03:56, 12.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 71/3000 [00:09<03:38, 13.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 73/3000 [00:09<04:58,  9.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▎         | 75/3000 [00:10<07:11,  6.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 78/3000 [00:10<07:56,  6.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 82/3000 [00:10<05:12,  9.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 84/3000 [00:11<04:32, 10.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 88/3000 [00:11<03:47, 12.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 92/3000 [00:11<04:17, 11.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 94/3000 [00:12<05:31,  8.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 100/3000 [00:12<05:30,  8.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▎         | 106/3000 [00:13<04:22, 11.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▎         | 108/3000 [00:13<04:27, 10.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▎         | 110/3000 [00:14<05:50,  8.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 113/3000 [00:14<05:16,  9.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 118/3000 [00:14<05:00,  9.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 120/3000 [00:15<05:58,  8.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 125/3000 [00:16<06:31,  7.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 131/3000 [00:16<04:01, 11.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 134/3000 [00:16<04:00, 11.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 136/3000 [00:17<07:21,  6.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 138/3000 [00:17<07:07,  6.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 140/3000 [00:17<07:00,  6.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▍         | 145/3000 [00:18<05:04,  9.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 148/3000 [00:18<05:40,  8.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 151/3000 [00:19<05:57,  7.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 154/3000 [00:19<07:00,  6.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▌         | 159/3000 [00:20<05:41,  8.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▌         | 163/3000 [00:20<04:39, 10.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 166/3000 [00:20<03:47, 12.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 168/3000 [00:20<04:05, 11.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 171/3000 [00:21<05:03,  9.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 173/3000 [00:21<05:15,  8.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 175/3000 [00:22<06:46,  6.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 177/3000 [00:22<05:48,  8.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 179/3000 [00:22<05:02,  9.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 185/3000 [00:22<04:14, 11.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 187/3000 [00:23<05:00,  9.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▋         | 193/3000 [00:23<04:34, 10.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▋         | 195/3000 [00:24<05:50,  8.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 200/3000 [00:24<05:26,  8.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 205/3000 [00:25<04:13, 11.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 207/3000 [00:25<05:37,  8.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 210/3000 [00:25<04:43,  9.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 212/3000 [00:26<06:29,  7.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 214/3000 [00:26<06:38,  6.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 216/3000 [00:27<07:30,  6.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 222/3000 [00:27<03:52, 11.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 224/3000 [00:27<03:36, 12.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 226/3000 [00:27<04:20, 10.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 228/3000 [00:28<06:25,  7.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 232/3000 [00:28<06:01,  7.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      " {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 236/3000 [00:29<05:15,  8.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 238/3000 [00:29<04:45,  9.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 244/3000 [00:29<03:33, 12.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 246/3000 [00:30<06:51,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 250/3000 [00:30<05:42,  8.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 252/3000 [00:31<05:37,  8.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 254/3000 [00:31<05:42,  8.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▊         | 258/3000 [00:31<04:44,  9.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▊         | 260/3000 [00:32<05:33,  8.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2026-01-13 12:43:14.781\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▊         | 262/3000 [00:40<1:03:16,  1.39s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:43:16.783\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a domain expert in functional genomics. For experiments carried out in K562 cells, we perturb PTDSS1 and examine the expression of ZNF341. Does perturbing PTDSS1 lead to a significant change in ZNF341 expression? Reply only with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are a domain expert in functional genomics. For experiments carried out in K562 cells, we perturb PTDSS1 and examine the expression of ZNF341. Does perturbing PTDSS1 lead to a significant change in ZNF341 expression? Reply only with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 3771}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 5 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 263/3000 [00:42<1:08:07,  1.49s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:43:16.823\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 4 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:43:17.249\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to P4HB and then measure expression of RP11-430H10.1. Does this perturbation cause a significant change in RP11-430H10.1 expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to P4HB and then measure expression of RP11-430H10.1. Does this perturbation cause a significant change in RP11-430H10.1 expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 5024}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 4 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 265/3000 [00:43<48:56,  1.07s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:43:18.115\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert analyst of perturbation datasets. For K562 cells subjected to perturbation of PTDSS1, does the expression profile of RNF19A indicate a significant change relative to control conditions? Reply strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are an expert analyst of perturbation datasets. For K562 cells subjected to perturbation of PTDSS1, does the expression profile of RNF19A indicate a significant change relative to control conditions? Reply strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 3767}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 3 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 266/3000 [00:44<47:09,  1.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:43:18.120\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in gene regulation studies. For experiments performed in K562 cells, CARS is perturbed and WARS expression is measured. Determine whether WARS exhibits a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are an expert in gene regulation studies. For experiments performed in K562 cells, CARS is perturbed and WARS expression is measured. Determine whether WARS exhibits a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 4889}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 3 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:43:18.141\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in gene regulation studies. For experiments performed in K562 cells, MRPL39 is perturbed and MANF expression is measured. Determine whether MANF exhibits a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are an expert in gene regulation studies. For experiments performed in K562 cells, MRPL39 is perturbed and MANF expression is measured. Determine whether MANF exhibits a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 2419}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 3 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:43:20.266\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a domain expert in functional genomics. For experiments carried out in K562 cells, we perturb COPZ1 and examine the expression of STARD9. Does perturbing COPZ1 lead to a significant change in STARD9 expression? Reply only with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are a domain expert in functional genomics. For experiments carried out in K562 cells, we perturb COPZ1 and examine the expression of STARD9. Does perturbing COPZ1 lead to a significant change in STARD9 expression? Reply only with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 4601}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 269/3000 [00:46<40:32,  1.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:43:20.919\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 3 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 270/3000 [00:47<38:37,  1.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:43:21.377\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 271/3000 [00:47<35:01,  1.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:43:21.541\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are assisting with the interpretation of perturbation-based expression data. In K562 cells, SEL1L is perturbed and ZMYND8 expression is quantified. Does this perturbation result in a significant change in ZMYND8 expression compared with control cells? Respond exactly as 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are assisting with the interpretation of perturbation-based expression data. In K562 cells, SEL1L is perturbed and ZMYND8 expression is quantified. Does this perturbation result in a significant change in ZMYND8 expression compared with control cells? Respond exactly as 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 1522}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 4 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 272/3000 [00:47<28:49,  1.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:43:21.551\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to MTHFD1 and then measure expression of PKDCC. Does this perturbation cause a significant change in PKDCC expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to MTHFD1 and then measure expression of PKDCC. Does this perturbation cause a significant change in PKDCC expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 1794}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 4 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:43:21.656\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in regulatory genomics. Consider data from K562 cells in which TTI2 is perturbed and EP300 expression is observed. Does this perturbation lead to a significant difference in EP300 expression relative to control conditions? Answer strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are an expert in regulatory genomics. Consider data from K562 cells in which TTI2 is perturbed and EP300 expression is observed. Does this perturbation lead to a significant difference in EP300 expression relative to control conditions? Answer strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 2053}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 4 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 274/3000 [00:47<18:35,  2.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:43:21.730\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in regulatory genomics. Consider data from K562 cells in which CCND3 is perturbed and RP1-274L7.1 expression is observed. Does this perturbation lead to a significant difference in RP1-274L7.1 expression relative to control conditions? Answer strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are an expert in regulatory genomics. Consider data from K562 cells in which CCND3 is perturbed and RP1-274L7.1 expression is observed. Does this perturbation lead to a significant difference in RP1-274L7.1 expression relative to control conditions? Answer strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 4073}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 3 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:43:21.746\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in gene expression analysis. For K562 cells, assess whether perturbation of SYVN1 is associated with a significant change in UBE2V1 expression compared with unperturbed controls. Answer strictly as 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are an expert in gene expression analysis. For K562 cells, assess whether perturbation of SYVN1 is associated with a significant change in UBE2V1 expression compared with unperturbed controls. Answer strictly as 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 1705}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 3 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:43:21.761\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a domain expert in functional genomics. For experiments carried out in K562 cells, we perturb DDRGK1 and examine the expression of IL8. Does perturbing DDRGK1 lead to a significant change in IL8 expression? Reply only with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are a domain expert in functional genomics. For experiments carried out in K562 cells, we perturb DDRGK1 and examine the expression of IL8. Does perturbing DDRGK1 lead to a significant change in IL8 expression? Reply only with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 1251}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 3 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 277/3000 [00:47<10:50,  4.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:43:21.838\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in regulatory genomics. Consider data from K562 cells in which TARS is perturbed and RP11-499F3.2 expression is observed. Does this perturbation lead to a significant difference in RP11-499F3.2 expression relative to control conditions? Answer strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are an expert in regulatory genomics. Consider data from K562 cells in which TARS is perturbed and RP11-499F3.2 expression is observed. Does this perturbation lead to a significant difference in RP11-499F3.2 expression relative to control conditions? Answer strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 2883}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 3 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:43:21.909\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in gene regulation studies. For experiments performed in K562 cells, FARSB is perturbed and ACE expression is measured. Determine whether ACE exhibits a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are an expert in gene regulation studies. For experiments performed in K562 cells, FARSB is perturbed and ACE expression is measured. Determine whether ACE exhibits a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 2579}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 3 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 279/3000 [00:48<08:34,  5.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:43:21.920\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in single-cell biology and functional genomics. In K562 cells, DDOST is perturbed and the expression of TRPM4 is measured. Does this perturbation cause a significant change in TRPM4 expression? Answer strictly in the format 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are an expert in single-cell biology and functional genomics. In K562 cells, DDOST is perturbed and the expression of TRPM4 is measured. Does this perturbation cause a significant change in TRPM4 expression? Answer strictly in the format 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 3840}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 3 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:43:21.937\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in single-cell biology and functional genomics. In K562 cells, SPCS3 is perturbed and the expression of CFH is measured. Does this perturbation cause a significant change in CFH expression? Answer strictly in the format 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are an expert in single-cell biology and functional genomics. In K562 cells, SPCS3 is perturbed and the expression of CFH is measured. Does this perturbation cause a significant change in CFH expression? Answer strictly in the format 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 5050}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 3 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:43:23.412\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 282/3000 [00:49<14:07,  3.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:43:23.940\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:43:23.959\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 284/3000 [00:50<13:38,  3.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:43:24.258\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 285/3000 [00:50<13:37,  3.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:43:24.835\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in single-cell biology and functional genomics. In K562 cells, KCTD16 is perturbed and the expression of CCDC69 is measured. Does this perturbation cause a significant change in CCDC69 expression? Answer strictly in the format 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are an expert in single-cell biology and functional genomics. In K562 cells, KCTD16 is perturbed and the expression of CCDC69 is measured. Does this perturbation cause a significant change in CCDC69 expression? Answer strictly in the format 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 3440}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 286/3000 [00:50<15:57,  2.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:43:24.890\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:43:24.902\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:43:25.049\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 289/3000 [00:51<10:15,  4.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:43:25.069\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:43:25.108\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to SAMM50 and then measure expression of MARCH8. Does this perturbation cause a significant change in MARCH8 expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to SAMM50 and then measure expression of MARCH8. Does this perturbation cause a significant change in MARCH8 expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 1434}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 5 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:43:25.516\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 3 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 292/3000 [00:51<08:59,  5.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:43:25.689\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 3 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 293/3000 [00:51<08:47,  5.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:43:25.721\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 3 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:43:26.876\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert analyst of perturbation datasets. For K562 cells subjected to perturbation of DHDDS, does the expression profile of RP11-304M2.5 indicate a significant change relative to control conditions? Reply strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are an expert analyst of perturbation datasets. For K562 cells subjected to perturbation of DHDDS, does the expression profile of RP11-304M2.5 indicate a significant change relative to control conditions? Reply strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 4537}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 3 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 295/3000 [00:53<14:20,  3.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:43:27.061\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 296/3000 [00:53<13:15,  3.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:43:27.077\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:43:27.082\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:43:27.097\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:43:28.390\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 300/3000 [00:54<14:06,  3.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:43:28.550\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 301/3000 [00:54<13:02,  3.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:43:28.970\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 302/3000 [00:55<14:05,  3.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:43:29.050\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:43:29.853\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 304/3000 [00:55<16:02,  2.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:43:29.863\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:43:29.998\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are assisting with the interpretation of perturbation-based expression data. In K562 cells, HSPA5 is perturbed and TSC22D4 expression is quantified. Does this perturbation result in a significant change in TSC22D4 expression compared with control cells? Respond exactly as 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are assisting with the interpretation of perturbation-based expression data. In K562 cells, HSPA5 is perturbed and TSC22D4 expression is quantified. Does this perturbation result in a significant change in TSC22D4 expression compared with control cells? Respond exactly as 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 3272}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 4 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 306/3000 [00:56<11:50,  3.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:43:30.191\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 307/3000 [00:56<11:15,  3.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:43:30.205\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:43:30.222\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:43:30.230\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:43:30.232\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:43:30.251\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:43:30.291\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:43:30.316\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in regulatory genomics. Consider data from K562 cells in which ASCC3 is perturbed and ATP5EP2 expression is observed. Does this perturbation lead to a significant difference in ATP5EP2 expression relative to control conditions? Answer strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are an expert in regulatory genomics. Consider data from K562 cells in which ASCC3 is perturbed and ATP5EP2 expression is observed. Does this perturbation lead to a significant difference in ATP5EP2 expression relative to control conditions? Answer strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 2953}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 3 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 314/3000 [00:56<04:24, 10.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:43:30.365\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:43:30.652\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 317/3000 [00:57<07:38,  5.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2026-01-13 12:43:31.523\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 3 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:43:31.779\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 3 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 319/3000 [00:57<07:56,  5.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:43:32.121\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:43:33.694\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in regulatory genomics. Consider data from K562 cells in which DDOST is perturbed and TNFRSF10B expression is observed. Does this perturbation lead to a significant difference in TNFRSF10B expression relative to control conditions? Answer strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are an expert in regulatory genomics. Consider data from K562 cells in which DDOST is perturbed and TNFRSF10B expression is observed. Does this perturbation lead to a significant difference in TNFRSF10B expression relative to control conditions? Answer strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 3863}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 321/3000 [00:59<16:05,  2.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:43:33.775\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in regulatory genomics. Consider data from K562 cells in which HSPA9 is perturbed and PPP4R2 expression is observed. Does this perturbation lead to a significant difference in PPP4R2 expression relative to control conditions? Answer strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are an expert in regulatory genomics. Consider data from K562 cells in which HSPA9 is perturbed and PPP4R2 expression is observed. Does this perturbation lead to a significant difference in PPP4R2 expression relative to control conditions? Answer strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 2373}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 323/3000 [01:01<18:59,  2.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 325/3000 [01:02<19:55,  2.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 332/3000 [01:02<07:49,  5.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 334/3000 [01:02<06:45,  6.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█▏        | 338/3000 [01:03<05:06,  8.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█▏        | 340/3000 [01:03<04:34,  9.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█▏        | 343/3000 [01:04<07:49,  5.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 347/3000 [01:04<06:14,  7.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 349/3000 [01:04<05:10,  8.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 353/3000 [01:05<04:57,  8.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 356/3000 [01:05<03:49, 11.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 358/3000 [01:05<05:25,  8.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 360/3000 [01:06<07:48,  5.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 364/3000 [01:06<06:30,  6.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 366/3000 [01:06<05:29,  8.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 369/3000 [01:07<05:36,  7.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 371/3000 [01:07<04:35,  9.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 376/3000 [01:07<02:47, 15.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 378/3000 [01:08<04:47,  9.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 382/3000 [01:08<05:27,  8.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 384/3000 [01:09<06:36,  6.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 387/3000 [01:09<07:00,  6.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 390/3000 [01:09<04:57,  8.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 393/3000 [01:10<04:02, 10.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 398/3000 [01:10<04:26,  9.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 401/3000 [01:10<03:47, 11.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 403/3000 [01:11<07:23,  5.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▎        | 407/3000 [01:11<05:20,  8.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▎        | 409/3000 [01:12<06:29,  6.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 413/3000 [01:12<05:20,  8.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 420/3000 [01:13<03:01, 14.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 422/3000 [01:13<05:01,  8.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 424/3000 [01:14<06:26,  6.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 426/3000 [01:14<06:46,  6.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 432/3000 [01:14<04:22,  9.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 434/3000 [01:15<05:03,  8.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▍        | 438/3000 [01:15<04:23,  9.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 440/3000 [01:15<04:16,  9.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 442/3000 [01:16<05:05,  8.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 444/3000 [01:16<06:34,  6.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 446/3000 [01:16<05:25,  7.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 448/3000 [01:17<06:31,  6.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 451/3000 [01:17<05:28,  7.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 453/3000 [01:17<05:18,  7.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 456/3000 [01:18<05:11,  8.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 457/3000 [01:18<05:10,  8.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 463/3000 [01:18<04:15,  9.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 465/3000 [01:19<05:13,  8.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 467/3000 [01:19<06:55,  6.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 475/3000 [01:20<04:12, 10.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 477/3000 [01:20<04:55,  8.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 480/3000 [01:20<04:20,  9.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 482/3000 [01:21<04:42,  8.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 483/3000 [01:21<05:13,  8.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 486/3000 [01:21<05:35,  7.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▋        | 489/3000 [01:21<04:18,  9.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▋        | 491/3000 [01:22<03:51, 10.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▋        | 493/3000 [01:22<04:07, 10.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▋        | 495/3000 [01:22<04:54,  8.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 497/3000 [01:22<05:55,  7.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 498/3000 [01:23<06:48,  6.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81242b050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81249e6d0>, 2339451.963864714)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81242b890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8124361d0>\n",
      "Evaluating workflow:  17%|█▋        | 499/3000 [01:23<08:54,  4.68it/s]Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c812441310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c812443450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81249e970>, 2339451.908408169)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c812436990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81249ed60>, 2339452.054124154)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c812441b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c812455850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8124630d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c812474bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81249e200>, 2339452.316852811)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c812474a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c812477a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c812481d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c812483c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c812491fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8124936d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8124a1410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8124a3410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81249db00>, 2339452.043840375)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c812443b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811f45850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81249ef20>, 2339452.110138775)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c812455710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81249dfd0>, 2339452.46105636)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8124635d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81249e4a0>, 2339452.592759795)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c812477950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81249f230>, 2339452.04543067)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c812482050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81249e820>, 2339451.953054803)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c812483b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81249eac0>, 2339451.719667228)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8125d3e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81249de10>, 2339452.377915558)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c812493b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81249fc40>, 2339452.163835006)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c812411ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c812414c90>, 2339454.11225039)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c88bbdae10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811e9e190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e52890>, 2339454.277738503)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8124d29d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811e4ddd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811cfdb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c812416190>, 2339454.35442603)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c88bfdf990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811f35490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d38360>, 2339454.368905202)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811d46bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811d9c4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d39e10>, 2339455.28501695)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811d9c290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c812416040>, 2339453.85282351)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811e4e050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811a37490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e52dd0>, 2339456.531199584)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811a37350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811a65710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811a3cfa0>, 2339457.323177074)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811a66fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811d836d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811a3e0b0>, 2339456.45544078)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811d83910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8124c89d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d38130>, 2339456.722087354)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81241a250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811a9da90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81193aa50>, 2339459.205052437)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811a9ce50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811cfe4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81193b690>, 2339458.923984477)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811cfe590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8118fe590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81182c830>, 2339459.592524508)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8118ff110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8118b1d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81182f930>, 2339460.454312821)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8118b2fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8118c0150>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8117284b0>, 2339460.44975579)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8118c3f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81181d710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8117290f0>, 2339461.197079577)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81178ca90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8117c7950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8118595d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8118c1a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81182f070>, 2339459.448792368)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811889ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81182d160>, 2339459.445874151)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c88bfc9a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811620690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811628280>, 2339461.972016465)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811620950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8116b5c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81162ac80>, 2339462.327324215)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8116b5f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8112f5e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81162bee0>, 2339462.810703251)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8112f5d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81165b690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811629630>, 2339461.429571105)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81165b490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8115f9d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811204360>, 2339464.750337338)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811665850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8113674d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811729da0>, 2339463.420240768)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811367390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81133d210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811207540>, 2339466.368797927)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8116ab510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811138950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811204670>, 2339466.174636013)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811138890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811108690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811682dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110f99b0>, 2339466.744733378)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8111092d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110f9b70>, 2339466.423429115)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811682310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811136d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110fbaf0>, 2339466.87439962)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811136910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811175cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8111ec750>, 2339467.245197902)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811176190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8113ee4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811206dd0>, 2339467.454106384)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811012310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8111e9d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8111ec1a0>, 2339466.731759529)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8111e9b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810f3d590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81182d860>, 2339469.032489023)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810f3cbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811d9b910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8110aedd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110eecf0>, 2339519.325083426)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810ef3b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8116b7450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811849950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c88c1f0e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c88bfdf2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8112d0c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810f69850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810f10450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c88bf9e190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110efc40>, 2339518.081043296)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8125e8150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8115e0050>, 2339518.35402557)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811870310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110ef310>, 2339518.63350788)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811849850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110ef460>, 2339518.774196287)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c88c1f1110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110ef620>, 2339518.379913173)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c88bfdd110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110ef150>, 2339523.786666658)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8112d0fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c88b0bc590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8115e1630>, 2339518.64069985)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c88b0bc910>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c88b0c6e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c88b0cea50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c88b0d26d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c88b0d9750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8115e1d30>, 2339518.020135651)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c88b0d9610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8115e17f0>, 2339518.507601534)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c88b0c7310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8115e1a90>, 2339518.681366604)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810f4ea90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8115e1400>, 2339518.59696772)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c88b0d3810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8115d7790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8115ddad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8115df190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8115e2350>, 2339518.820516697)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c88b0dc4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8115e20b0>, 2339518.366255731)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8115dd850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810cf0310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810ec72a0>, 2339520.57800837)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8117fbb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810eea0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e52740>, 2339521.122756252)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810ee9210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810d26150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810d74590>, 2339520.781317496)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810eab190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810d5fd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810d75320>, 2339521.441192191)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810d5ff90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810de6490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810d76eb0>, 2339522.213309222)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810f3e9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810d443d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8111ee9e0>, 2339522.343402018)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810d444d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810d30d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810d77690>, 2339522.297139732)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810d30d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81081f2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810828d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810d77d20>, 2339522.86842753)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81081ddd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810d77ee0>, 2339522.813322814)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810828b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810810790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810852910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110ed240>, 2339522.311198123)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810d263d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81086ef90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c9880dd810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810831390>, 2339522.817818475)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c9880ddad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810894990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810d76e40>, 2339523.339064521)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8108945d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810836710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8108ab210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810831cc0>, 2339523.324115397)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810daf3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8108325f0>, 2339523.658907012)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8108ab0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8108c4d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810832a50>, 2339523.458010159)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810d1d2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810ea1c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8108d3590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810ec6350>, 2339522.87733214)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810d9ae10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d39c50>, 2339522.73601884)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81086c9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810832d60>, 2339523.606461839)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8108c6810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8108332a0>, 2339523.651694551)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8108d3a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810705c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81249f380>, 2339451.510464562)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c812443ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811f37010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811e27f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811e5a550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c812442ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811e8f450>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c812457250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811e54e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811ee4750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811d11610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811d2c450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811d44a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811d5b690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811d827d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811cff810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811d56890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811f362d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811e67090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8119f6010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811a59d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811a5f7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811a8d9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811a9ccd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811abed90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8118fdd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811914a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811ab7a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811d5bf10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811a74090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8118f5510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8119517d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81196f690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8119af190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8119de6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8119bfe50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81181e910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811825b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81184c650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811873210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81196db90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811878750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8118e4650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8118ed910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81170f4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81172c8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811873d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811de4f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811729390>, 2339462.539505509)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8117b6e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8118b0990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811734e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8117e6810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8115fa050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81160ed90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811dacb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811872150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811695a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8116caf10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8116ebf10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8118feed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811694050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8116e9110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8116aaad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81131a6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8113426d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811384710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811398fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8115f9ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811681950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8113ecb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81120f5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811226c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8113e11d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81124f310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81126ced0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8113e9bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81120a450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8112abed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811351150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8112d3910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81115d510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8110f27d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8111887d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8111bdd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811283cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8111030d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81174add0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8111ddc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81102a950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811056050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81107c1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811057910>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8110acd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8110d9fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810ef1ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810efaad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81104f390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8110aec50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810f2c210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8115e1080>, 2339518.591988684)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810f68d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8115e1240>, 2339518.437596072)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810f11c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8115e0ec0>, 2339518.26485695)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c88bf9c450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c88c490a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810f4f710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8115e2510>, 2339518.14469029)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c88b0bf250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810e9fad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810eca6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810ecf650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810edcfd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810cf4d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810eab550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810d08d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810d1d010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810d1ec50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810d3b450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810d5de90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810d6a6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810d8ed50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810d9bf50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810daf190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810833b60>, 2339523.532631028)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810705b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8124cf2a0>, 2339454.150520276)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8124a3a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e33c40>, 2339453.821323404)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811e27dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c812517460>, 2339453.82553841)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811e5aa10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81249c6e0>, 2339453.353007184)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c812442c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e502f0>, 2339455.419374233)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811e8fb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e51be0>, 2339455.193535673)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c812456e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e526d0>, 2339454.667393489)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811e54dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e52a50>, 2339454.950905502)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811ee4c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e53540>, 2339454.613366328)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811d114d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e53e70>, 2339454.908903835)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811d1ba10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d380c0>, 2339454.558827644)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811f3d010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d38980>, 2339454.260794507)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811d5b4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d391d0>, 2339454.735196449)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811d82690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8124cc210>, 2339455.83711273)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8124646d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d38b40>, 2339455.280566005)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811d33810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d3a120>, 2339457.036463218)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811dbfbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d3ac80>, 2339455.851690618)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811e67050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d39cc0>, 2339457.581545445)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8119f5e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811a3c980>, 2339456.488133316)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811a5a3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811a3cec0>, 2339456.418748579)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811a5fd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e52040>, 2339456.400891266)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811a8d7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811a3db70>, 2339456.813975216)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811a9cb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811a3e900>, 2339457.230689132)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811abeb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811a3f460>, 2339456.784436738)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8118fdbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811a3fa80>, 2339457.189929002)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811914ed0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811a3c280>, 2339456.729029662)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811ab78d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d38280>, 2339457.884116277)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811d5b8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811938130>, 2339457.594567019)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811d9e550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8119382f0>, 2339457.911721456)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8118f5590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811938bb0>, 2339457.954749738)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811ab1c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811939470>, 2339458.290268079)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81196fb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81193a820>, 2339459.173605622)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8119ad850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81249cbb0>, 2339458.720483474)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8119de4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81193b3f0>, 2339458.56455832)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8119ebf50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81182c0c0>, 2339459.640966365)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81181f010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81182c280>, 2339459.151298952)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811826010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81182ca60>, 2339459.361176916)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81184cd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81193b9a0>, 2339459.155235969)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811873890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8119392b0>, 2339457.891815835)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81196d710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81182db70>, 2339459.339590666)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811878f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8119386e0>, 2339461.126505987)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8118dba50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81182fbd0>, 2339460.221028189)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8118ed550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81182fd90>, 2339460.667840937)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81170fc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81193ae40>, 2339460.649991693)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8118b6a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811729be0>, 2339461.156826518)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811873b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81182d320>, 2339462.397951608)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811870ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81182cde0>, 2339460.154146275)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8118b0790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811728750>, 2339460.673081446)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811735ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81193a9e0>, 2339462.110172053)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8117e6fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811a3e740>, 2339462.274386994)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8115f9c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811628050>, 2339461.564134473)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81160d850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811628910>, 2339462.135653095)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8118b5690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811628ad0>, 2339461.655786611)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81163a890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81162a890>, 2339462.95697973)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8118ed850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81162ae40>, 2339462.696918672)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8116cb5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81162bd20>, 2339463.445228597)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8116ebe10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81182e190>, 2339461.857072496)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811658610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8119390f0>, 2339463.637446436)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811694610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81162ba80>, 2339463.350779301)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8116e9890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811729b70>, 2339463.41014221)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8117143d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811314b40>, 2339464.136372374)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8112ffc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811315160>, 2339463.707537778)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811343dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81172bee0>, 2339464.205853256)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81168ae10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811316580>, 2339464.177440274)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811398e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811314440>, 2339464.193764173)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8115fa510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81182c750>, 2339464.176267975)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811681d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d3b690>, 2339465.297996689)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8113ec950>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8112048a0>, 2339464.362275351)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81120f3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811204ec0>, 2339464.45158689)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811227110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811205240>, 2339464.699939151)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8113e0050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81182d8d0>, 2339464.704611669)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81139dbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811205c50>, 2339464.785271946)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81121d450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811317a80>, 2339464.397942229)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8113e9810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811205f60>, 2339465.462151825)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81120a090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81162a120>, 2339465.721551882)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8112abcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811207150>, 2339465.330409022)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811350e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8112077e0>, 2339465.434532027)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8112d3710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110f9da0>, 2339466.679041839)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8118da110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110fa2e0>, 2339466.584766242)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8110f2890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811938ad0>, 2339466.307656427)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811188e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110f9160>, 2339467.179133476)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8111bff10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d39b00>, 2339466.237589139)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81129b490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811314a60>, 2339465.851128497)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811102ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110faf90>, 2339467.05040089)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811748ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110fb930>, 2339468.165170643)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8112b6210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811204280>, 2339467.28647124)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81102a490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8111ee200>, 2339467.113406872)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811055f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8111ee970>, 2339467.923836462)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81107f150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8111ed470>, 2339467.781854125)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811276f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8116295c0>, 2339468.1104863)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8110ad090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110ec130>, 2339468.516123632)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8110db710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8116280c0>, 2339469.554742319)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810ef18d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110ece50>, 2339469.468888935)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810efbf90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8111edc50>, 2339467.887034114)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81104f890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8111eec80>, 2339468.237810578)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8110aec10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110ed550>, 2339468.45439654)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810f2c890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110efe00>, 2339518.923516952)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810f58150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110eef90>, 2339518.857419823)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810f4fd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810ec4050>, 2339520.342332937)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c88c212250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8115e2e40>, 2339520.280031435)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810ec9e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810ec4a60>, 2339520.618928078)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810ecfb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110ed160>, 2339520.750875297)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810edd490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810ec5b00>, 2339520.223975484)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810cf4b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810ec5d30>, 2339520.245248785)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810eaba90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810ec5ef0>, 2339520.328193272)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810d09390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c88bbe4de0>, 2339520.4038042)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c88b0c5050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810ec69e0>, 2339520.916092125)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810d1d250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810ec7460>, 2339520.462916083)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810d3b250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810ec7cb0>, 2339521.031617269)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810d5dd50>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8115e1860>, 2339520.989385308)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810d698d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110ef9a0>, 2339521.782528296)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8115df250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810d750f0>, 2339521.322437056)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810d9be50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810d758d0>, 2339521.781064832)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810daf050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81073c4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81073f550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810861890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8111edbe0>, 2339524.267659283)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81073d550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811204ad0>, 2339524.424776304)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81073f3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110fa190>, 2339524.777717164)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81070c4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8112d9450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8116294e0>, 2339524.811688039)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8112db190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81086ed90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811374550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81172ba10>, 2339525.328354646)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810fb7490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810ec6510>, 2339524.499114652)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811377a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810802290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81182e3c0>, 2339525.366872834)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810d32790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81188bb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d3a4a0>, 2339525.939866873)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811889c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8117a6a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811a445d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c812414440>, 2339524.7163607)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8110e2bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810831a20>, 2339525.131811445)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810ddc890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811e8d4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8124ce200>, 2339525.500807018)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811e8d3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81132ce10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8124cf1c0>, 2339525.294206452)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811602ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811dd4a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c88bbe50f0>, 2339525.269243013)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811f34890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c88bc5fa90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c88bbe7d90>, 2339525.260903001)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c88bc5e810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81085a890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c88c076690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810831780>, 2339525.451943394)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810802ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c88bbe40c0>, 2339525.474219475)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c88c074f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c88c374b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c88c228510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c88c453e00>, 2339525.680919402)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c88c374110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c88c4534d0>, 2339526.08121681)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8125d0c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810daff10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81249eb30>, 2339526.521454722)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810715b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81079c210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810744280>, 2339526.47077935)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81074ff90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81079d8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810744c20>, 2339526.915776343)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810dad910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8112fe750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8107cb7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810745240>, 2339527.992944681)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8112fc690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110ec590>, 2339527.248477713)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8107cb2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8105f07d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810830ad0>, 2339527.049183538)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8116e8d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8107d50d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8107460b0>, 2339527.05361192)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81070d350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8106298d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810747000>, 2339527.187383602)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81062a190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811e8d0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8125e9ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810746c10>, 2339527.292112723)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811e8d850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810747a80>, 2339527.663240274)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8125e9f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810881550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810670210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110ee510>, 2339527.367072638)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810653790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8108304b0>, 2339527.498618753)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810670890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81067ba10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c88c2bf9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810674ad0>, 2339527.436313301)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81067bd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c88c453620>, 2339525.58922012)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811209fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810686550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81069a910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8107472a0>, 2339527.41985131)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8106864d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810d77d90>, 2339528.468952717)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810698450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810641bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810675470>, 2339528.077047755)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810641cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8106e0410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810676350>, 2339528.107925556)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8106e0ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8106c3a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810831320>, 2339527.958550796)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8125df150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8106ce4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81051c0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810675b00>, 2339528.376813685)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8106ce410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8115e0830>, 2339528.473318655)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810643e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81050b790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810676ba0>, 2339529.150831634)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81050a9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810565390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810d77f50>, 2339529.188286631)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810565210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810530dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811a198d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810554f30>, 2339529.248020728)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8107cbb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8105b1090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8111ee120>, 2339531.609273828)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811ec1610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8105c1290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810d77310>, 2339529.235523541)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8105c1610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8105c36d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8105ea110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8106c26d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810554d70>, 2339529.3273488)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810578190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810675e10>, 2339527.429486478)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8106c1e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810554600>, 2339529.67166143)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8125ea690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e50280>, 2339529.705122942)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8105e9f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8103fa590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810556ba0>, 2339529.759004802)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8106181d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8107d7490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810428390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810556f90>, 2339529.659702579)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81057b490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8106b8890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810746430>, 2339529.986559269)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810692310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810509390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8104845d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810830830>, 2339530.406302916)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810501410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810458e50>, 2339531.238084704)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810484d10>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810487b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810459550>, 2339530.335762738)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810487c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810489bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8104b9ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810459630>, 2339529.900656598)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810489e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810459be0>, 2339530.430999672)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8104b9cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8104cec50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8105b10d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81045a200>, 2339530.707371351)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810463390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8108c4a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810409950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810745010>, 2339530.668388871)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81030b410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81045b690>, 2339531.133288774)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810408c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8102f5bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810344410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810408290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810d77b60>, 2339529.788999411)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810423f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81045a040>, 2339530.407529388)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8104ceb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810348050>, 2339531.172274595)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81062bb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810348210>, 2339532.78754701)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810344ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810556dd0>, 2339529.287110101)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810408950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810351a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810555320>, 2339531.877963878)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8103513d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8103a44d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810349080>, 2339533.001845208)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8103a5d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81069b450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8107470e0>, 2339531.605835204)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81069bc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8103b0cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8103cd690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810349cc0>, 2339532.379587377)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8103b0bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81034a430>, 2339533.038823731)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8103cd490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810372950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8101f8f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81034a510>, 2339532.384451524)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810372d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8105552b0>, 2339532.686224864)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8101f9610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810204390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8104596a0>, 2339532.424572263)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810494590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81021fcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810676f20>, 2339532.486678631)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81021c590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810218f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8104600d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8102671d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81025c8a0>, 2339532.696764807)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810219190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81025c750>, 2339532.736438111)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810462c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81025cb40>, 2339532.618206913)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810267090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810281e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81045ac10>, 2339532.816519422)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810281cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810295dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810304610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81025d780>, 2339532.7192092)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810296510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8103486e0>, 2339531.117595942)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810306e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8102bf610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810484750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81025e270>, 2339534.091118711)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810484fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81057aad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8102e3110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81025d080>, 2339533.906467103)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8102c6690>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81025eb30>, 2339533.664806531)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8102e3b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810530410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81025edd0>, 2339535.617176329)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81059e510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8102a5390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810132490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81025f3f0>, 2339533.821256126)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8102a4ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8102969d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810267450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81034b230>, 2339534.732731367)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81013f510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e53070>, 2339534.698305637)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810267110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81018ee10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810141860>, 2339534.860245968)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81018ecd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81019d110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810140600>, 2339534.838969381)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81019d550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8101b3b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8101d8f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810140830>, 2339534.647402228)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81046a650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810142c80>, 2339534.344517659)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8101d9390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81015a450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3f066d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81025da90>, 2339534.369833729)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8102bf4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810747bd0>, 2339534.574688044)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810132290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810142e40>, 2339534.77210066)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81015a290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810143000>, 2339535.710349464)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3f06590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810176610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810143310>, 2339535.67576509)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810176590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3f39f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3f1b110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810747380>, 2339535.202763334)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8103286d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3f48750>, 2339536.242511638)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3f04f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8103448d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3f904d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810142270>, 2339536.577072782)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810345a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3f49470>, 2339536.553302958)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8101a7410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8101d9510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3fb4790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3f48d00>, 2339536.675556909)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8101d9cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3fcea50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810459da0>, 2339536.835859646)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3fce9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3ff35d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3f4af90>, 2339536.203052389)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3ff33d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3fcced0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3e17710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81025fb60>, 2339537.108237481)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8101b3450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3f4bb60>, 2339536.965273657)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3e175d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3f6df10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81025cde0>, 2339537.291800743)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810396890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3e3a3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3e5aad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3f2f610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3f48a60>, 2339536.238039058)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3fb4c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810143cb0>, 2339535.248685474)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3f2f250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3f265d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810348520>, 2339537.01463865)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3e3a1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810143700>, 2339536.904375715)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3e5afd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81045bbd0>, 2339535.223141632)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3f26b90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3ea4510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e51ef0>, 2339537.352878824)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3ea4850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3e72c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3e48050>, 2339537.320556016)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810264ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3e40d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3ee8310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3e4a200>, 2339537.39884191)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3e40e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3eeb250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8101434d0>, 2339537.927054658)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3eeb910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3ec9e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3e4b310>, 2339537.753277111)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3eca6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810298910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810557d90>, 2339538.470764682)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3d1fa90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3efa850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3d3fc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3d442f0>, 2339538.74209818)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3d0ce90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3d444b0>, 2339538.292877045)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3d3fb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3d7f150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81025d320>, 2339538.697055008)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3d7f7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3d7f450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3d45550>, 2339538.609360358)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8101bd7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3d94910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3db7710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3e6b1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810142ac0>, 2339538.002435183)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3ed7bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810142190>, 2339537.024338584)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81043b210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81025dd30>, 2339539.11455567)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3d94d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3f4a2e0>, 2339538.511367359)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3db75d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3ffa990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3e49080>, 2339536.783191168)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3e78850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3dc2550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3d46c10>, 2339539.19536036)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3d626d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3ff0910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3d45080>, 2339539.305474589)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3ff0bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3e73f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3d44750>, 2339539.245266462)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3e70690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3c02ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3c25a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3d47e00>, 2339539.658207459)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3c02f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3d47cb0>, 2339540.410711613)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3c26150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3d7e390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3c41c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3c282f0>, 2339540.156022078)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3d7d110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3c28750>, 2339540.226909929)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3c41a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3dd2ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3c57650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3e499b0>, 2339539.294751802)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3dd3910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3c28f30>, 2339539.601868493)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3c57f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3c6f350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3c29550>, 2339539.271093077)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3c6f210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3c826d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3c29cc0>, 2339539.623615296)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3ea4e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3c96450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3cb89d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3e4b380>, 2339540.148847406)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3f2ded0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3c2a660>, 2339540.272699922)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3cb9390>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3ccd510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3cee550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3f48590>, 2339540.701052738)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3ca5750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3d44a60>, 2339540.237877228)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3ceea50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3b013d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3c2baf0>, 2339541.331149221)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3b011d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3ee8550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3b250d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3c2be70>, 2339540.854615006)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3ee96d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3b282f0>, 2339541.200158531)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3dc2e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3ce1b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3b28590>, 2339541.48267634)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3ce1290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3b6b690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3b79b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3f4baf0>, 2339541.670645639)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3b6b890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3c28d70>, 2339541.488554545)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3b7b1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3b872d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3b2a430>, 2339541.62746268)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3b87190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3ba60d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3e489f0>, 2339541.144205753)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3ba6810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3b3fdd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3bb7f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3b2ac10>, 2339541.233421598)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3d1d850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3b2ae40>, 2339541.327049912)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3bb7e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3c613d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3bdbc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3b2b3f0>, 2339541.31344668)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3c61050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3f4b690>, 2339542.242263581)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3bdbad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3b424d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3b2b8c0>, 2339542.127456283)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3b9ff10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3cef050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3d40490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a14ad0>, 2339544.055336129)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3cef290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a14d00>, 2339542.139695919)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3d41050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3a023d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3a647d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a15320>, 2339542.747216258)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3a02490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8101410f0>, 2339542.043541612)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3a66790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3a70090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a15ef0>, 2339543.052648733)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3a70750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3aa0c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3ab7dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a16820>, 2339542.840653908)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3aa0c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3f4a660>, 2339543.51328457)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3ab7c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3bc91d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a179a0>, 2339543.601390121)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3bc8b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3ab26d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3913650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3b288a0>, 2339543.673091522)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3ab2350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3c2af90>, 2339543.891506738)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d39131d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3932dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3c29470>, 2339543.488401517)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d39307d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3ae6390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d394aa50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d390cf30>, 2339543.678774173)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3ae7890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d390d0f0>, 2339543.359262612)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d394a7d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d395c4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d396a290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810674980>, 2339544.165341977)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3c14750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d390d7f0>, 2339543.692703845)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d396a150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3983dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d390e200>, 2339543.768509251)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3983cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3924e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3c2b000>, 2339543.882071675)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3925dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d39bda90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3a72310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d390f070>, 2339544.173958308)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d39bdf90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a144b0>, 2339542.266664864)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3a72150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3bec0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d39d5c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3d45b00>, 2339545.100407613)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d39aa6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a16f90>, 2339564.239283045)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d39d5a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d39a7e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d39f6410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d390eb30>, 2339544.675792493)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d39a4910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d390fb60>, 2339546.146952643)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d39f6b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d380f390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d382d050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3b2bbd0>, 2339545.756591291)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d380f610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a16b30>, 2339545.216980792)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d382d850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d383d950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3809470>, 2339545.257521169)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d383dfd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d394b750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3809ef0>, 2339545.101543694)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d394a810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3831cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d38a4e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d380aa50>, 2339545.18441077)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3831b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d380ac80>, 2339546.054138907)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d38a5590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d38bc790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d390dfd0>, 2339545.54542879)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d38bc5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d380d5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d38d4750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d380b930>, 2339545.64601774)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d38bed50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d380bb60>, 2339546.210322725)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d38d5850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d38e60d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d380bd20>, 2339545.565914127)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d384e350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d38e7590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3709a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a16270>, 2339545.804673272)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d38577d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d38f4a60>, 2339546.048205998)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3708c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d371a890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d38f4c20>, 2339546.465580627)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d39cae10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d388f610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3739910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3d45a90>, 2339546.074051775)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3728c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d38f5400>, 2339547.255603724)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3912350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3706110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d38f55c0>, 2339546.073008541)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3706210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3763dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d38f5780>, 2339547.500924304)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3763d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d378d6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3a70850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d38f6a50>, 2339546.947599609)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d378c310>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d38f6c10>, 2339547.749286706)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3832a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d37a1110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d37cd1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d38f7930>, 2339547.367265958)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d37a12d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d38f7b60>, 2339547.261586742)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d37cd550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3867790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d380af20>, 2339547.286289883)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3b021d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3611490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d38098d0>, 2339559.29135069)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3611bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d37bf7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3632210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d36009f0>, 2339547.603992356)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d37bf9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3601320>, 2339547.863644563)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3632590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d373b790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3601be0>, 2339548.190887312)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3706190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3ea5950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d390e190>, 2339560.622147527)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d380dad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3633d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3602a50>, 2339547.918701143)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3632690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3665850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d38a57d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3602c80>, 2339548.517823571)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3666f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d390d9b0>, 2339548.115460698)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d38a61d0>\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3626990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d36dc9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d36038c0>, 2339548.247138574)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3732f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d36fc650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3500440>, 2339562.77521134)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d36fc9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3505850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d38089f0>, 2339560.627788738)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3505f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3539d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3603540>, 2339564.05889185)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3539b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3507650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d35507d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3501940>, 2339564.050405113)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d35077d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3501b70>, 2339564.165234782)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3768810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d356ad50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d35025f0>, 2339559.757861711)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d356ab90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d356f850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3502900>, 2339564.429207819)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d356f710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3584e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d35030e0>, 2339564.346912186)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3585910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d35b26d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d36814d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d380a580>, 2339548.553332443)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d36dd2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3602820>, 2339548.015819126)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3680f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3503690>, 2339564.270056289)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d35b3b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d365e750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d35e4210>, 2339564.41826818)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d369c110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3220310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d390c280>, 2339559.331923121)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3220590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d327d650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d35e6c10>, 2339564.446575719)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d327d450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3762d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3260c20>, 2339567.617014633)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d329c710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d35f9610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3205ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d328f2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d35e48a0>, 2339560.650096963)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d35f9c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3c29a20>, 2339557.28837405)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3205d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3603b60>, 2339567.343904979)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d377b190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d35ef0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d30d0490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3603bd0>, 2339564.254669564)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d35eef90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d30bd320>, 2339563.427480006)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d35f8210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2fa2150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2f32c10>, 2339566.767346908)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2fa2b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2fc8650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2f33000>, 2339569.384553445)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2fc9010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2e3a3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2e014e0>, 2339567.578503572)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2e3ab50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2f32970>, 2339568.025080666)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2e6d850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2f145d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2f294d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2fa8390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2f32dd0>, 2339572.824376221)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2fa8990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2ff93d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d30bfbd0>, 2339566.448003154)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2f14d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2f30280>, 2339566.467921017)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2f28f10>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2e00210>, 2339565.920744601)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2ff3cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2f65850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2f31470>, 2339563.88614676)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2f144d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2f14f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2f316a0>, 2339568.227949376)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2f590d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2e9fed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2e03700>, 2339567.411488134)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2e9fdd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2fced10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3501e10>, 2339572.507011974)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2fced90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2d68390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d38f5080>, 2339569.589966783)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2d681d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2d99ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ef2dd0>, 2339571.05859505)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2d9a5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2f64cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d30d3f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2ed9490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2d2f890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2d43cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3260d00>, 2339568.198042074)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2e3ac10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2e00440>, 2339576.284433617)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d30d36d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ef0130>, 2339576.203158034)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2ed9350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ef1d30>, 2339567.557685011)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2d24150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d36015c0>, 2339569.569411099)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2d43ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2ebd650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2e03930>, 2339567.398917367)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2ebd450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2dd3f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2dbcad0>, 2339569.606524952)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2dd3e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2c0b150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2dbdb70>, 2339570.898529447)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2dfb190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2c7d590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3263540>, 2339571.558839224)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2c7d410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2cece10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2c99e80>, 2339572.873828626)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2cecc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2b233d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2c9b0e0>, 2339573.160785907)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2b239d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2b49310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3a6efd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2c30810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2c5cfd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2ccf7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3262970>, 2339572.361936519)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2b49bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2e02120>, 2339569.583752165)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d36a8f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2dbea50>, 2339571.47912701)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2c30f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2dbf8c0>, 2339577.449722362)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2c5d9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2c99ef0>, 2339572.799063168)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2ccf5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2cbe590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2c99be0>, 2339572.699747637)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2c32e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2decd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2dbcf30>, 2339572.371102449)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2decd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3206d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2dbf380>, 2339572.737082908)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3206fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2becc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2b671c0>, 2339572.72862559)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2bed390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2bba2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3601010>, 2339572.713787819)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2bb9e10>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 501/3000 [01:23<07:51,  5.30it/s]\r",
      "Evaluating workflow:  17%|█▋        | 502/3000 [01:24<07:07,  5.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 505/3000 [01:24<04:28,  9.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 509/3000 [01:24<03:45, 11.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 513/3000 [01:24<03:18, 12.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      " {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 516/3000 [01:25<04:06, 10.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 521/3000 [01:25<04:58,  8.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 523/3000 [01:26<06:00,  6.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 529/3000 [01:26<03:54, 10.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 531/3000 [01:26<03:31, 11.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 533/3000 [01:27<03:50, 10.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 535/3000 [01:27<04:33,  9.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 537/3000 [01:27<05:30,  7.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 539/3000 [01:28<09:04,  4.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 545/3000 [01:29<05:16,  7.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 551/3000 [01:29<04:06,  9.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▊        | 556/3000 [01:29<03:45, 10.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▊        | 558/3000 [01:30<07:19,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▊        | 562/3000 [01:31<05:29,  7.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 567/3000 [01:31<04:26,  9.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 570/3000 [01:31<03:40, 11.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 572/3000 [01:32<04:14,  9.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 576/3000 [01:32<04:42,  8.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 578/3000 [01:33<05:09,  7.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 579/3000 [01:33<07:16,  5.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 582/3000 [01:33<05:36,  7.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 584/3000 [01:34<06:06,  6.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:44:20.627\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a domain expert in functional genomics. For experiments carried out in K562 cells, we perturb STT3A and examine the expression of RCBTB2. Does perturbing STT3A lead to a significant change in RCBTB2 expression? Reply only with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are a domain expert in functional genomics. For experiments carried out in K562 cells, we perturb STT3A and examine the expression of RCBTB2. Does perturbing STT3A lead to a significant change in RCBTB2 expression? Reply only with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 1331}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 585/3000 [01:46<1:25:24,  2.12s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:44:20.660\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in gene expression analysis. For K562 cells, assess whether perturbation of SRPR is associated with a significant change in RP11-146D12.2 expression compared with unperturbed controls. Answer strictly as 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are an expert in gene expression analysis. For K562 cells, assess whether perturbation of SRPR is associated with a significant change in RP11-146D12.2 expression compared with unperturbed controls. Answer strictly as 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 475}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:44:20.726\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in gene regulation studies. For experiments performed in K562 cells, COPB1 is perturbed and PDRG1 expression is measured. Determine whether PDRG1 exhibits a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are an expert in gene regulation studies. For experiments performed in K562 cells, COPB1 is perturbed and PDRG1 expression is measured. Determine whether PDRG1 exhibits a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 4159}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:44:21.174\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert analyst of perturbation datasets. For K562 cells subjected to perturbation of TMED10, does the expression profile of RP11-242O24.5 indicate a significant change relative to control conditions? Reply strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are an expert analyst of perturbation datasets. For K562 cells subjected to perturbation of TMED10, does the expression profile of RP11-242O24.5 indicate a significant change relative to control conditions? Reply strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 3997}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 7 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 588/3000 [01:47<53:47,  1.34s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:44:21.309\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 7 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 589/3000 [01:47<45:58,  1.14s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:44:22.406\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in gene regulation studies. For experiments performed in K562 cells, TTI2 is perturbed and EP300 expression is measured. Determine whether EP300 exhibits a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are an expert in gene regulation studies. For experiments performed in K562 cells, TTI2 is perturbed and EP300 expression is measured. Determine whether EP300 exhibits a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 2059}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 7 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 590/3000 [01:48<45:35,  1.14s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:44:22.500\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a functional genomics specialist. In K562 cells, we perturb COPB1 and monitor FGF11 expression. Decide whether this perturbation leads to a significant alteration in FGF11 expression. Answer only in the format 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are a functional genomics specialist. In K562 cells, we perturb COPB1 and monitor FGF11 expression. Decide whether this perturbation leads to a significant alteration in FGF11 expression. Answer only in the format 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 4178}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 7 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:44:22.733\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in gene expression analysis. For K562 cells, assess whether perturbation of SCYL1 is associated with a significant change in RHCE expression compared with unperturbed controls. Answer strictly as 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are an expert in gene expression analysis. For K562 cells, assess whether perturbation of SCYL1 is associated with a significant change in RHCE expression compared with unperturbed controls. Answer strictly as 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 3505}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 7 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 592/3000 [01:48<31:57,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:44:22.775\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in single-cell biology and functional genomics. In K562 cells, TIMM44 is perturbed and the expression of EP400 is measured. Does this perturbation cause a significant change in EP400 expression? Answer strictly in the format 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are an expert in single-cell biology and functional genomics. In K562 cells, TIMM44 is perturbed and the expression of EP400 is measured. Does this perturbation cause a significant change in EP400 expression? Answer strictly in the format 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 3660}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 7 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:44:24.861\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a functional genomics specialist. In K562 cells, we perturb IDH3A and monitor VIMP expression. Decide whether this perturbation leads to a significant alteration in VIMP expression. Answer only in the format 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are a functional genomics specialist. In K562 cells, we perturb IDH3A and monitor VIMP expression. Decide whether this perturbation leads to a significant alteration in VIMP expression. Answer only in the format 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 978}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 5 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 594/3000 [01:50<35:29,  1.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:44:24.872\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a functional genomics specialist. In K562 cells, we perturb DDRGK1 and monitor UBE3A expression. Decide whether this perturbation leads to a significant alteration in UBE3A expression. Answer only in the format 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are a functional genomics specialist. In K562 cells, we perturb DDRGK1 and monitor UBE3A expression. Decide whether this perturbation leads to a significant alteration in UBE3A expression. Answer only in the format 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 1228}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 5 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:44:28.873\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 596/3000 [01:55<49:56,  1.25s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:44:28.998\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in gene regulation studies. For experiments performed in K562 cells, DAD1 is perturbed and HLA-AS1 expression is measured. Determine whether HLA-AS1 exhibits a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are an expert in gene regulation studies. For experiments performed in K562 cells, DAD1 is perturbed and HLA-AS1 expression is measured. Determine whether HLA-AS1 exhibits a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 769}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 597/3000 [01:55<41:30,  1.04s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:44:29.492\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 598/3000 [01:55<36:55,  1.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:44:29.537\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:44:29.577\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in gene expression analysis. For K562 cells, assess whether perturbation of CCND3 is associated with a significant change in NTRK1 expression compared with unperturbed controls. Answer strictly as 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are an expert in gene expression analysis. For K562 cells, assess whether perturbation of CCND3 is associated with a significant change in NTRK1 expression compared with unperturbed controls. Answer strictly as 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 4035}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:44:29.698\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in regulatory genomics. Consider data from K562 cells in which FECH is perturbed and GALNT7 expression is observed. Does this perturbation lead to a significant difference in GALNT7 expression relative to control conditions? Answer strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are an expert in regulatory genomics. Consider data from K562 cells in which FECH is perturbed and GALNT7 expression is observed. Does this perturbation lead to a significant difference in GALNT7 expression relative to control conditions? Answer strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 4383}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 5 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 601/3000 [01:55<20:40,  1.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:44:29.762\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:44:29.874\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 603/3000 [01:56<15:19,  2.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:44:29.912\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:44:30.006\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 605/3000 [01:56<11:24,  3.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:44:30.031\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to FECH and then measure expression of RP11-157D23.2. Does this perturbation cause a significant change in RP11-157D23.2 expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to FECH and then measure expression of RP11-157D23.2. Does this perturbation cause a significant change in RP11-157D23.2 expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 4424}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 4 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:44:30.040\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are assisting with the interpretation of perturbation-based expression data. In K562 cells, EIF2B3 is perturbed and LINC01137 expression is quantified. Does this perturbation result in a significant change in LINC01137 expression compared with control cells? Respond exactly as 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are assisting with the interpretation of perturbation-based expression data. In K562 cells, EIF2B3 is perturbed and LINC01137 expression is quantified. Does this perturbation result in a significant change in LINC01137 expression compared with control cells? Respond exactly as 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 4662}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 4 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:44:30.108\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are assisting with the interpretation of perturbation-based expression data. In K562 cells, TMED10 is perturbed and RP11-242O24.5 expression is quantified. Does this perturbation result in a significant change in RP11-242O24.5 expression compared with control cells? Respond exactly as 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are assisting with the interpretation of perturbation-based expression data. In K562 cells, TMED10 is perturbed and RP11-242O24.5 expression is quantified. Does this perturbation result in a significant change in RP11-242O24.5 expression compared with control cells? Respond exactly as 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 3992}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 4 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 608/3000 [01:56<07:24,  5.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:44:30.158\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a domain expert in functional genomics. For experiments carried out in K562 cells, we perturb AMIGO3 and examine the expression of SNHG12. Does perturbing AMIGO3 lead to a significant change in SNHG12 expression? Reply only with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are a domain expert in functional genomics. For experiments carried out in K562 cells, we perturb AMIGO3 and examine the expression of SNHG12. Does perturbing AMIGO3 lead to a significant change in SNHG12 expression? Reply only with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 841}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 4 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:44:31.349\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 3 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 610/3000 [01:57<12:09,  3.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:44:31.577\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a functional genomics specialist. In K562 cells, we perturb SPCS2 and monitor GATA2 expression. Decide whether this perturbation leads to a significant alteration in GATA2 expression. Answer only in the format 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are a functional genomics specialist. In K562 cells, we perturb SPCS2 and monitor GATA2 expression. Decide whether this perturbation leads to a significant alteration in GATA2 expression. Answer only in the format 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 5118}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 3 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:44:34.247\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 612/3000 [02:00<24:58,  1.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:44:34.429\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 613/3000 [02:00<22:00,  1.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:44:34.952\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a domain expert in functional genomics. For experiments carried out in K562 cells, we perturb ASCC3 and examine the expression of CPQ. Does perturbing ASCC3 lead to a significant change in CPQ expression? Reply only with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are a domain expert in functional genomics. For experiments carried out in K562 cells, we perturb ASCC3 and examine the expression of CPQ. Does perturbing ASCC3 lead to a significant change in CPQ expression? Reply only with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 2961}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 617/3000 [02:01<12:36,  3.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 619/3000 [02:01<09:37,  4.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2026-01-13 12:44:35.482\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 621/3000 [02:01<08:20,  4.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 623/3000 [02:02<11:43,  3.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 625/3000 [02:02<09:38,  4.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 627/3000 [02:03<07:51,  5.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 631/3000 [02:03<03:55, 10.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 633/3000 [02:04<08:22,  4.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 635/3000 [02:04<07:22,  5.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██▏       | 640/3000 [02:04<05:02,  7.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 646/3000 [02:05<02:54, 13.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 651/3000 [02:05<03:34, 10.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 653/3000 [02:06<04:44,  8.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 655/3000 [02:06<06:08,  6.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 659/3000 [02:07<05:02,  7.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 664/3000 [02:07<04:00,  9.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 666/3000 [02:07<04:46,  8.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 667/3000 [02:08<05:12,  7.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 672/3000 [02:08<04:02,  9.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 678/3000 [02:09<04:05,  9.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 681/3000 [02:09<03:29, 11.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 683/3000 [02:09<04:16,  9.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 685/3000 [02:10<05:00,  7.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 687/3000 [02:10<05:19,  7.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 692/3000 [02:10<04:08,  9.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 694/3000 [02:11<05:57,  6.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 699/3000 [02:12<05:12,  7.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 701/3000 [02:12<04:37,  8.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▎       | 706/3000 [02:12<03:29, 10.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▎       | 708/3000 [02:12<03:56,  9.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▎       | 712/3000 [02:13<03:57,  9.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 714/3000 [02:13<05:15,  7.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 718/3000 [02:14<05:04,  7.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 724/3000 [02:14<03:15, 11.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 726/3000 [02:15<03:53,  9.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 728/3000 [02:15<06:03,  6.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 733/3000 [02:16<04:16,  8.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 735/3000 [02:16<03:45, 10.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 737/3000 [02:16<04:35,  8.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 739/3000 [02:16<04:36,  8.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▍       | 744/3000 [02:17<04:14,  8.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 746/3000 [02:17<05:17,  7.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▌       | 751/3000 [02:18<04:07,  9.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 753/3000 [02:18<04:42,  7.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▌       | 756/3000 [02:19<05:00,  7.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 758/3000 [02:19<05:50,  6.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 760/3000 [02:19<05:12,  7.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 764/3000 [02:20<04:01,  9.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 768/3000 [02:20<03:29, 10.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 770/3000 [02:20<03:09, 11.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 772/3000 [02:21<06:00,  6.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 775/3000 [02:21<04:51,  7.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 777/3000 [02:21<05:21,  6.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 780/3000 [02:22<05:17,  6.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 782/3000 [02:22<05:13,  7.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 786/3000 [02:22<03:39, 10.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▋       | 790/3000 [02:23<04:59,  7.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▋       | 794/3000 [02:23<03:56,  9.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 796/3000 [02:24<06:11,  5.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 800/3000 [02:24<04:35,  7.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 804/3000 [02:25<03:51,  9.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 806/3000 [02:25<03:36, 10.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 811/3000 [02:25<03:29, 10.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 813/3000 [02:26<04:48,  7.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 819/3000 [02:26<03:54,  9.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 821/3000 [02:27<04:08,  8.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 826/3000 [02:27<03:44,  9.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 830/3000 [02:28<03:27, 10.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 832/3000 [02:28<04:59,  7.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 834/3000 [02:28<05:02,  7.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 840/3000 [02:29<03:50,  9.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 842/3000 [02:29<03:21, 10.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 846/3000 [02:30<04:06,  8.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 848/3000 [02:30<03:36,  9.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 850/3000 [02:30<03:27, 10.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 854/3000 [02:30<03:39,  9.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▊       | 856/3000 [02:31<06:02,  5.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▊       | 862/3000 [02:32<04:16,  8.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 866/3000 [02:32<02:56, 12.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 870/3000 [02:32<03:17, 10.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 872/3000 [02:33<04:20,  8.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 876/3000 [02:33<04:39,  7.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 878/3000 [02:34<04:48,  7.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 881/3000 [02:34<05:16,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2026-01-13 12:45:17.656\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 4 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 882/3000 [02:43<54:22,  1.54s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:45:17.668\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 4 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:45:17.774\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 4 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 884/3000 [02:43<38:27,  1.09s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:45:21.020\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 11 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 885/3000 [02:47<51:58,  1.47s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:45:21.095\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 10 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:45:21.116\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 10 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 888/3000 [02:47<29:56,  1.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 890/3000 [02:47<22:11,  1.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:45:21.678\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 10 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:45:21.723\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 10 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 892/3000 [02:47<17:44,  1.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:45:21.772\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 10 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:45:31.167\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in single-cell biology and functional genomics. In K562 cells, ZNF326 is perturbed and the expression of CEBPG is measured. Does this perturbation cause a significant change in CEBPG expression? Answer strictly in the format 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are an expert in single-cell biology and functional genomics. In K562 cells, ZNF326 is perturbed and the expression of CEBPG is measured. Does this perturbation cause a significant change in CEBPG expression? Answer strictly in the format 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 70}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 4 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:45:31.168\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to SPCS3 and then measure expression of GPR146. Does this perturbation cause a significant change in GPR146 expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to SPCS3 and then measure expression of GPR146. Does this perturbation cause a significant change in GPR146 expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 5084}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 4 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 894/3000 [02:57<1:01:32,  1.75s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:45:31.280\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in gene expression analysis. For K562 cells, assess whether perturbation of IER3IP1 is associated with a significant change in PDE4D expression compared with unperturbed controls. Answer strictly as 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are an expert in gene expression analysis. For K562 cells, assess whether perturbation of IER3IP1 is associated with a significant change in PDE4D expression compared with unperturbed controls. Answer strictly as 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 515}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 3 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 896/3000 [02:57<43:48,  1.25s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:45:31.288\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a domain expert in functional genomics. For experiments carried out in K562 cells, we perturb SRP68 and examine the expression of ZNF292. Does perturbing SRP68 lead to a significant change in ZNF292 expression? Reply only with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are a domain expert in functional genomics. For experiments carried out in K562 cells, we perturb SRP68 and examine the expression of ZNF292. Does perturbing SRP68 lead to a significant change in ZNF292 expression? Reply only with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 3561}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 3 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:45:31.546\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in regulatory genomics. Consider data from K562 cells in which COPB1 is perturbed and FGF11 expression is observed. Does this perturbation lead to a significant difference in FGF11 expression relative to control conditions? Answer strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are an expert in regulatory genomics. Consider data from K562 cells in which COPB1 is perturbed and FGF11 expression is observed. Does this perturbation lead to a significant difference in FGF11 expression relative to control conditions? Answer strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 4173}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 3 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 898/3000 [02:57<32:05,  1.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2026-01-13 12:45:31.797\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 3 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 900/3000 [02:57<23:48,  1.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:45:32.023\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in regulatory genomics. Consider data from K562 cells in which TTI1 is perturbed and RP11-16E12.1 expression is observed. Does this perturbation lead to a significant difference in RP11-16E12.1 expression relative to control conditions? Answer strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are an expert in regulatory genomics. Consider data from K562 cells in which TTI1 is perturbed and RP11-16E12.1 expression is observed. Does this perturbation lead to a significant difference in RP11-16E12.1 expression relative to control conditions? Answer strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 3233}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 3 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 901/3000 [02:58<20:59,  1.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:45:32.036\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to HSD17B12 and then measure expression of CDK13. Does this perturbation cause a significant change in CDK13 expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to HSD17B12 and then measure expression of CDK13. Does this perturbation cause a significant change in CDK13 expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 2764}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 3 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:45:34.241\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 903/3000 [03:00<26:55,  1.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:45:34.255\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a genomics expert evaluating perturbation experiments. In K562 cells, KCTD16 is perturbed and the expression of ARHGAP6 is measured. Determine whether ARHGAP6 shows a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are a genomics expert evaluating perturbation experiments. In K562 cells, KCTD16 is perturbed and the expression of ARHGAP6 is measured. Determine whether ARHGAP6 shows a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 3426}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:45:34.266\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:45:34.286\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:45:34.818\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 908/3000 [03:00<14:25,  2.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 909/3000 [03:01<15:52,  2.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|███       | 913/3000 [03:03<12:36,  2.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 915/3000 [03:03<09:54,  3.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 920/3000 [03:03<05:36,  6.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 923/3000 [03:03<04:12,  8.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 927/3000 [03:05<06:48,  5.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 935/3000 [03:05<03:37,  9.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 937/3000 [03:05<03:25, 10.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███▏      | 939/3000 [03:06<05:00,  6.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███▏      | 941/3000 [03:06<06:38,  5.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 947/3000 [03:07<03:44,  9.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 951/3000 [03:07<02:52, 11.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 953/3000 [03:07<02:46, 12.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 955/3000 [03:07<03:42,  9.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 957/3000 [03:08<03:55,  8.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 959/3000 [03:08<05:08,  6.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 965/3000 [03:09<03:55,  8.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 968/3000 [03:09<04:04,  8.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 972/3000 [03:10<04:03,  8.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 976/3000 [03:10<03:10, 10.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2a18890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2b67540>, 2339572.752807465)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2a181d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2a85810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2a36270>, 2339574.031767589)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2a85f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2bf4110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2aa5210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2b67380>, 2339572.73251052)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2bf4710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2a36cf0>, 2339574.287680897)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2aa5950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2df0750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ae0670>, 2339578.913122716)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2f15cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2af25d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ae0830>, 2339578.395845112)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2af23d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2931c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ae12b0>, 2339579.057681831)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2931f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2bebf50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d29d0590>, 2339578.577084657)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d298ae10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d29c7550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d29dbcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d29d07c0>, 2339579.652969798)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d29c7410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d29d0d70>, 2339578.927354185)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d29dbbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d295c8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2977390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d29ca090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ae2d60>, 2339578.667843829)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d295c610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ae3af0>, 2339578.992495025)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2977190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d29d0980>, 2339579.191286174)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2d25590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2888ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d29d3d90>, 2339579.508616654)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2888cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d28c7b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d288dd30>, 2339578.909350179)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d28c7a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2701450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d288ec10>, 2339579.522649325)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2a45650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d276b210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2744c20>, 2339578.9961157)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d276bb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d27b2d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2745b00>, 2339579.406848588)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d27b3710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2798c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2734710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2745400>, 2339579.723300469)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2799650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d288f930>, 2339579.745810236)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2735110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d288b2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2745fd0>, 2339581.033152597)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d271cf50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d269f450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d29d1630>, 2339581.233945305)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d27c7b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d28d26d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d27ec360>, 2339581.751915068)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d28d1810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2548810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d27ed710>, 2339582.783009403)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2548690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d25bffd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2a36900>, 2339583.225483973)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2567f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2b91d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d25d1b00>, 2339583.82721547)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2417610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d25dae10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2529510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d29d2820>, 2339583.36738106)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2529310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d26bc210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d27ed010>, 2339583.079482352)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d27bafd0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d288fee0>, 2339583.642976833)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d242a550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d24d2810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d24d48a0>, 2339586.354045989)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d24d2750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2462850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3502c10>, 2339584.476041122)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2462710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d25e3d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d25d0750>, 2339583.766132306)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2703750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2532350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2363510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d23d43d0>, 2339586.218273631)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2356690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d23d4590>, 2339587.753664744)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2362550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2480e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d23d5d30>, 2339587.143947331)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d223b350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2356890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2208f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2219810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d23d4de0>, 2339586.152992799)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2209610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d24d6120>, 2339585.638014115)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2356f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d23d5470>, 2339586.388971839)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d221ba50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d24d2590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d24d6580>, 2339588.376913699)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d24d2790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2130610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d22c5cc0>, 2339589.483269145)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2130510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d22c8ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d22c6f20>, 2339589.311756348)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d21753d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d202a310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d23d7310>, 2339589.173749607)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d202a250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2148990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d22c52b0>, 2339588.151702087)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2148fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d26f9ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d21bf3f0>, 2339590.900403742)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d22b63d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1f2aad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d22c70e0>, 2339591.74721209)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1f29c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d21d7490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1fc6740>, 2339592.543681393)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d204f250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1e5edd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ea4c20>, 2339593.581710801)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1e5fa50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1fe4910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1f55590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d20be0b0>, 2339591.998314926)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1fd7a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1fc5400>, 2339591.861809851)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1f55410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1c61250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1fbc790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1d58a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d20bd8d0>, 2339593.901296277)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1eb4b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d20bd390>, 2339596.082329242)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1c61890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ea6cf0>, 2339593.560773331)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1d590d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1b26fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1b940c0>, 2339598.072771822)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1e787d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1b73810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ea70e0>, 2339597.147958191)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1b72a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1bbe990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1b95400>, 2339598.522141715)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1bbee90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1a0cf50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1b97230>, 2339598.38961261)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1a0d810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d193dc10>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1a86ba0>, 2339600.523331668)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d193dad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d196a4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1a87540>, 2339600.418081667)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d195b190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d19794d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1a87e70>, 2339600.311522517)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d19799d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d197b010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1b950f0>, 2339600.732079131)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d197b490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d19a1650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d198ca60>, 2339600.863009768)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d19a1810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1ba16d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d198d240>, 2339600.781748837)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1ba2510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d19d2690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1c63710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d198d940>, 2339600.496860933)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d19da6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d198c360>, 2339600.469498193)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d19d2290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d19fb810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d198e190>, 2339600.99393561)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d19fb690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d19d1c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1829c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d198e740>, 2339600.615457582)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d19d1d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d198e900>, 2339600.992850163)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1829a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1839250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d198f230>, 2339601.170125196)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1839050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1ac96d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d198f540>, 2339601.123895103)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1ac9e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1849690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d187d6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d198f700>, 2339601.561235205)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1849450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d198fd90>, 2339601.209706952)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d187dd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1a9dd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1a86190>, 2339600.84080734)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1821610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1af9910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2ad3c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2a20150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2803a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d282a690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d27e6490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2b93ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2741b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2815c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2712290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d26892d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d269d350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2684290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d26d6010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2703fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2938290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d267c750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d26fb4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d255a6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2582050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d27f2190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2689b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d25f8d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2415090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d242f650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d25926d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2458890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d24988d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d249d350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d24c6b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2898410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d24f9c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2328d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d24d1510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2415f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d27d97d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2349610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d25f9d10>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d23e26d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d23eb010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2498610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d24faa90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d228b490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d22592d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d22c3410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2288210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2239550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d23cd190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d24ec1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d215e110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2162cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2192190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d21a89d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d21cbf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d228c550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d21f9890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d21c3890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d203de10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d23e3d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2128650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2069890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d20964d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d20a7450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d20c3d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d21741d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d21fbf50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1f3e6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1f6d350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1f7de50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1f92990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d25da6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1f01a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1fbc490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1f4b290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1e09c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1e1a610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1e4e090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1e84a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1e32d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d21fb6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1ecedd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1ec2ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1e9dfd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1d0d690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1f282d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d228bc50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1d1f090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1d83f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1d89f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1dc6850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1e0ae10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1c00c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1c20950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1c35a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d20444d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1c04b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1c6bd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d24ecdd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1c8cb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1cad510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d236e6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1b0f350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1b1f590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1d6c190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1da5450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1cee250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1ba0710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1b7e590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1c60850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1dda3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1b56050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1bd4b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1cae290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1a3b1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1a0e510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1a6e110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1a81f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1efa6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1aad390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1ac94d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1ae1350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1911110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1928210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1af8fd0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1a83c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ca1470>, 2339600.016807265)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1af9710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d188cf30>, 2339601.510639027)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1afa290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2a37ee0>, 2339574.630129691)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2abd850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2f32a50>, 2339573.885619486)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2a2d850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d29d1a90>, 2339579.278250356)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d29fae10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d29d22e0>, 2339579.210358359)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d282a510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ef08a0>, 2339580.555507102)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d27e59d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2b65400>, 2339581.548912014)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2742e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2b66f90>, 2339581.405871437)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2740ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2b642f0>, 2339580.876077719)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2815590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d29d0de0>, 2339580.992481987)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d284cf50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2747b60>, 2339581.557186786)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2689f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2745ef0>, 2339581.695673783)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d269da50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d27ec590>, 2339581.613754036)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3149550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d27ec130>, 2339581.612579367)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d26d5e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d29d3d20>, 2339582.464575568)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d271cd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2746660>, 2339580.917445776)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2938250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d29d19b0>, 2339581.100931931)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d267cd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d27ecc90>, 2339582.83466365)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d26fafd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d288d550>, 2339583.291369997)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d255a690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d27eeeb0>, 2339583.962879014)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d28f25d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ef1470>, 2339583.608951714)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2685410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ae04b0>, 2339583.955158649)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d268af90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d25d10f0>, 2339583.917414479)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d25f9110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d25d14e0>, 2339583.886548031)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d28a5d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d25d1cc0>, 2339584.028741969)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d242f450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d27ef460>, 2339582.78074369)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2592f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d29d2c80>, 2339583.884706393)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2458ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d25d3230>, 2339584.545251623)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2499010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d25d3540>, 2339585.116445508)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d249d810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d24d44b0>, 2339585.003644649)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d24aae10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d24d4670>, 2339585.194875779)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2898590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d25d3d20>, 2339585.082724339)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d24f9a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d24d56a0>, 2339585.647993807)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d25a1a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d24d5a20>, 2339585.493441348)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d24d0f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d27ec830>, 2339587.085297979)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d24163d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d24d72a0>, 2339586.197346717)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2338490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d24d4d00>, 2339587.216642879)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2349410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d24d7a80>, 2339585.954334982)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d25fa190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d23d47c0>, 2339586.518790849)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d23e2650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d24d6f90>, 2339586.472463834)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d272b050>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d23d5b00>, 2339586.29762762)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2499750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d23d6ac0>, 2339587.120209271)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d24f9d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d23d7150>, 2339587.060773729)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d228bc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d23d7460>, 2339588.198473316)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d28699d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d22c4590>, 2339588.218151466)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d22c30d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d25d2190>, 2339587.011203575)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d22888d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d27ef310>, 2339588.153867816)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2239b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d22c4750>, 2339587.29564386)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d239ab90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d22c6430>, 2339588.859282786)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d213ccd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d22c69e0>, 2339589.039243889)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d215e5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d22c6d60>, 2339589.031947328)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2162ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2747230>, 2339589.219267907)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d21877d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d25d2f20>, 2339588.818649376)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d21a90d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d22c7c40>, 2339589.27847436)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d223b5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d21bcd00>, 2339589.320658628)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2274650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d21bd5c0>, 2339589.403515786)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d21fbd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d21bdb70>, 2339589.366012588)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d22ac4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d23d5a20>, 2339590.031402044)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d203da10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d21bcfa0>, 2339589.334161919)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d24162d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d24d5400>, 2339589.476367606)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d205ffd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d21bf540>, 2339590.524758159)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d206a050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d21bfc40>, 2339591.266516116)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2096c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d21bff50>, 2339590.826185367)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d20a7250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d20bc600>, 2339590.540659304)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d20c3c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d22c71c0>, 2339591.342549373)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d20da210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d288c910>, 2339591.295165443)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d20eaa10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d22c6660>, 2339591.233632904)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1f3ed90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d20bc3d0>, 2339590.950857235)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1f6d150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d22c66d0>, 2339591.682308711)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1f7e350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d20bff50>, 2339591.587485107)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1f92610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d20bdda0>, 2339592.945771404)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2193a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1fc4280>, 2339592.020256518)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1f01290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d20be9e0>, 2339591.304985003)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1fbcb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1fc46e0>, 2339592.030675)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1f4b050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d24d6660>, 2339592.366988033)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1e0a110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1fc4f30>, 2339592.207603453)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1e1a410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1fc6890>, 2339593.107723528)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d208f210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1fc7540>, 2339593.024935515)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1e84890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1fc6200>, 2339593.669116601)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1e33b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1fc7d90>, 2339593.147944039)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d21f8f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d20bd0f0>, 2339594.643521365)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1ecc4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d25d0de0>, 2339593.852121643)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1ec2cd0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ea5470>, 2339593.6704513)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1e9e0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ae06e0>, 2339594.287155472)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1d0d4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ea5080>, 2339594.081190902)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1f322d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d20bf310>, 2339595.012564508)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1e30a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d23d5b70>, 2339594.661460665)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1d1fcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ea7cb0>, 2339594.46411866)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1d83e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1fc5630>, 2339594.642303302)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1d89e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1da8ad0>, 2339594.370305054)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1dc6710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1da9080>, 2339594.584459282)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1e0a910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1da9710>, 2339595.139439752)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1dd44d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1fc6510>, 2339595.191395754)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1c20810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1daa740>, 2339595.506426766)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1c35890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1da8600>, 2339595.075792375)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2044890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1da95c0>, 2339595.518020411)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1c04b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1dab770>, 2339596.037885866)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1c68e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1dabe70>, 2339596.481213887)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1c17c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1dabcb0>, 2339596.128772006)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1c8d250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ca0910>, 2339596.045501057)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1cadc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ca04b0>, 2339596.042210357)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1c97490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1fc4b40>, 2339596.389731921)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1b0f210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ca26d0>, 2339596.98357221)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1b1d850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ca0b40>, 2339596.893165142)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1d63cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ca30e0>, 2339596.925309874)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1cdbf90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ca32a0>, 2339597.628765745)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1ced950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1b94670>, 2339597.429760196)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1ba0510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1da9940>, 2339597.385672512)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1b7e350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ca0d00>, 2339596.571371488)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1c60790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ea6430>, 2339596.477161587)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1dda650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ca33f0>, 2339596.942275027)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1b55fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1b95320>, 2339597.666748341)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1bd5350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1b96ba0>, 2339597.940110561)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1cadb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ea40c0>, 2339598.255929104)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1a3b990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1b96f20>, 2339598.375147745)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1a0ed10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1b977e0>, 2339598.15481344)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1a6e850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1b97d90>, 2339598.613335193)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1cba410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1a84590>, 2339598.966197164)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1c82250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1a84750>, 2339598.156047503)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1aad190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1a85080>, 2339598.816361182)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1ac9450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1a85b70>, 2339599.344569773)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1afb190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1a85e80>, 2339598.945109063)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1911790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1fc7c40>, 2339599.60798102)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1928890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1b94fa0>, 2339600.014507107)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1a83a10>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2ad3f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d27dba90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d26bfd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2c9a5f0>, 2339579.467111874)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2ad3dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d288ef20>, 2339580.571931978)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d27dbd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d29d2c10>, 2339582.140563934)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d26bfe10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d17445d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d22c47c0>, 2339602.566033831)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1745a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d182b350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d288e430>, 2339603.308310521)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d17629d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1911f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c88b164f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2a37850>, 2339603.020425847)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d19121d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1e5d7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d198ed60>, 2339603.135963464)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1e5f910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d19d9650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2a357f0>, 2339603.177623555)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d19d8b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1d59610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d236fb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2a34520>, 2339603.088983659)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1dc4150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2a35240>, 2339603.098728883)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d23cecd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1aba9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d24d3850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d29d1400>, 2339603.868020832)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d187dd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d29d31c0>, 2339603.302609593)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d24d3910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2571310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d29d0d00>, 2339603.286550887)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d257aa90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2160590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d29a8690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d314c3d0>, 2339603.854447753)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c88b15f390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d32601a0>, 2339603.534926067)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d29a8c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2749c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2989610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3261710>, 2339603.746824277)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d27e9a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d32602f0>, 2339604.242964792)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d29888d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d29d9ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d285a7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d170d210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1717810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d18e29d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d195ae50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1b89550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2a37460>, 2339603.190330668)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c88b165450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810d74130>, 2339602.298893965)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d170ee90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1b96510>, 2339602.340648561)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1765650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3261ef0>, 2339605.129455259)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d29d89d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d32637e0>, 2339605.128299592)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2a226d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81162b2a0>, 2339602.240319581)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1715850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d24d7f50>, 2339602.246070826)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d17544d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1b97700>, 2339602.772077155)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1b8b990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d283fc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3261a20>, 2339605.895036868)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d283f190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c88e4edbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2a44bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d32609f0>, 2339605.442525029)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d173dbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d32608a0>, 2339605.076572415)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2a45850>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2bea6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d35e5a90>, 2339606.735515138)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2bea010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2a5ec50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d35e4520>, 2339605.518592086)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2aa5850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2a5d2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2a2fd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d35e7150>, 2339606.025076432)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1efae10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d29d35b0>, 2339605.805037939)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2a2fa10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d296d9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2a5a710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d30bec10>, 2339605.166702908)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d296d110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d30bd710>, 2339605.739322136)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2a5a610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2ab4e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2a35710>, 2339605.177482983)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1766ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2514b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d27880d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d30bfd90>, 2339605.146248175)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2516d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d30bf930>, 2339605.482229131)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1765850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d23f6790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d30bf700>, 2339605.19386603)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d18e26d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2a6ef90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2821b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3263e00>, 2339605.801137218)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2a86650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3260fa0>, 2339606.245942667)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2820390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2bb8710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3602510>, 2339606.265091351)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2bb8810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d242d090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2bd84d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d36010f0>, 2339606.364481305)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2d6bcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d36000c0>, 2339606.254162563)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2bd8310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2c8c290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d35e69e0>, 2339607.640612609)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2c8ca50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2b903d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2b96210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2db7ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1b94e50>, 2339608.128274716)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2b90d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2a36b30>, 2339607.435184497)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2b97610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d35e43d0>, 2339608.589798338)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2db47d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2c31a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3602270>, 2339607.580230974)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2c31950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2c5d890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2c41710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3601be0>, 2339608.294023474)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2c5f7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3601320>, 2339607.307762023)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2c40ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2b360d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2a36040>, 2339608.225549696)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2b363d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2b35610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2d703d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d30be350>, 2339607.514285745)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d286a510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d390e040>, 2339607.977370083)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2d716d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2b69810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d29e78d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d390d630>, 2339607.713444131)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2a5ce10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d18ae910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2d9b7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d390e740>, 2339607.997494543)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2b68b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d390c750>, 2339608.071245783)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d29e76d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d390e120>, 2339608.344793749)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d285bb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2c23ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d390c1a0>, 2339608.663453779)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2c22210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2d6bb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2d81dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d390d0f0>, 2339609.120810625)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2d68b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d390cde0>, 2339608.657416532)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2d82290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2b29b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d28170d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8101424a0>, 2339608.647909645)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2b29550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810141a90>, 2339609.370264837)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2816010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2c32f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2d9e2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2a346e0>, 2339609.452371342)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2c30c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2da5d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810141630>, 2339609.955636338)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2da5990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2dee490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8101438c0>, 2339627.369747238)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2deded0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2da4750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2cbb310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810143230>, 2339610.176714697)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2abe590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d390d400>, 2339609.977919813)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2cb8a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2cc2510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81034b1c0>, 2339610.167776326)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2cc0310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2b343d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d390f1c0>, 2339627.379188794)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2b34790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d31505d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2c7d090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d390f5b0>, 2339623.135976507)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2c143d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81034bcb0>, 2339609.981757518)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3150410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d390dcc0>, 2339608.12586393)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2c7f710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2fa3f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d36011d0>, 2339632.54873226)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2fa2310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2cf4f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810349710>, 2339632.54059611)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2cf7f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2ccd950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81034bee0>, 2339623.683102209)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2aac2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2ee1f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2e30690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810349240>, 2339631.506652549)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2ca4c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81034a510>, 2339632.617458515)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2e30cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2e3a390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810349e10>, 2339632.666504371)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2e093d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2e3ae90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2eb2690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810557bd0>, 2339625.009209342)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2e3bc50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2e9e8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8101431c0>, 2339624.915159024)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2a8d550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2d26ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810556a50>, 2339625.241848862)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2d25850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2f40550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2cbea50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2cf6e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810349be0>, 2339625.283567604)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2eb18d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2f218d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2d2c7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8103483d0>, 2339610.539205342)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2cf7210>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2cdb4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810556120>, 2339634.086153237)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2f43750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810348d70>, 2339623.169286176)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2c65850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8105563c0>, 2339632.085817483)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2ce3cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810554bb0>, 2339632.207517488)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2e9e910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3600f30>, 2339623.234943304)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2cdb150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2f40e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8105570e0>, 2339637.723471747)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2e61ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2f4cad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110efc40>, 2339637.838191753)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2fd9490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d314aa90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110ed400>, 2339637.45905972)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3149050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d32c5cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d328f450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d356ca90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d20bd470>, 2339623.81815321)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d32c6290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110ef690>, 2339631.381209123)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d328ead0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c88bc024a0>, 2339637.465096044)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d356f990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d35b0d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d390ed60>, 2339632.001020411)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3224c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d37e6710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ae0c20>, 2339637.587558841)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d37e5910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d365c150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2e6d450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d32c6410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d30d0590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d30d1990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c88bbe4bb0>, 2339637.719269138)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d353bb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3520c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ae2200>, 2339639.619588339)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d365efd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8105572a0>, 2339632.045843436)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2cbae10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8115e0e50>, 2339632.27131787)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2e48290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c88bbe4440>, 2339632.513863442)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d30d0510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2b67d20>, 2339637.489835387)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3523490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d35b1010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d311cdd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3682450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c88bc01710>, 2339632.382792141)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2d04990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c88bc016a0>, 2339632.420369633)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d31244d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2b64130>, 2339633.857469761)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3682210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d35a73d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c88bbe40c0>, 2339639.371844236)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3612190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d36664d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ae18d0>, 2339639.527966051)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2e6f7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d38c71d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c812416c80>, 2339639.267117861)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d38d78d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d386c790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8124172a0>, 2339639.638815814)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d38bd1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3807d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2dbf0e0>, 2339636.938299116)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3806250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d399bc50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d32c5850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3779c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d37614d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d38fd790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c812416cf0>, 2339638.888649561)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2ce2810>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c812414de0>, 2339639.111103835)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3761150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2dbd9b0>, 2339637.990861264)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d39d5b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8115e2c10>, 2339639.535246103)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2ff8210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c812415320>, 2339638.986525628)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d38fc050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d381b450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2dbd160>, 2339636.75554423)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d381b0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d383da90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8124144b0>, 2339639.6030277)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3982e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3613410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e531c0>, 2339641.253249285)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2d6ae10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3ae7350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e51320>, 2339639.711656911)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3ae4290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3a0bcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e52820>, 2339640.785648423)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3a087d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3bb2dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ef34d0>, 2339640.607281544)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3bb3d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3bfdd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e52580>, 2339641.400154594)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3b426d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3bca350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3bfde10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3b0c050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3a806d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ef0f30>, 2339640.936066268)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3bc9010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810557690>, 2339640.8979428)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2ffba10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2dbdcc0>, 2339641.074230458)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3b0ee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110ecad0>, 2339640.860147628)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3a80a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3cab950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ef2c80>, 2339641.362780652)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3cab350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3abf3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e50520>, 2339641.437744557)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3cb87d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3763890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c812414590>, 2339642.468567181)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2e49490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d381a510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3dd0150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ef0a60>, 2339641.25081852)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3818ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e52f20>, 2339641.963803997)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3dd0210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3c14e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e501a0>, 2339641.96725423)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3c15350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3dc0d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3d3ee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2c9af20>, 2339641.492688141)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3db64d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2c99be0>, 2339642.112303204)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3d3ec50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d394b550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8115e3770>, 2339641.995904569)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3ea5e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3d0fc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2c9b230>, 2339642.539905706)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d36dd4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3a23110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e52970>, 2339643.025206266)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3bb5a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3ed78d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3e68d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2dbff50>, 2339644.927923978)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3ef1110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e507c0>, 2339643.527539632)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3e6b8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3fcda50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3c44910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3ddf490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3daeb10>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810348a60>, 2339642.0052244)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3daf950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ef0ec0>, 2339643.516019544)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3afcdd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ef0600>, 2339641.262210492)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3c46cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ef3310>, 2339641.451038588)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3bb5390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3e33690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810554e50>, 2339643.301417785)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3e20d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3ecae10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ef0de0>, 2339643.490734994)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3ec95d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3e7c450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1a857f0>, 2339643.31911861)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3e7f5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3e784d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81019c890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ae3e00>, 2339643.152742447)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3ab5fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1a87930>, 2339643.511764794)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81019c950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3e17390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3ed4090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1a86200>, 2339644.442082928)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3939ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2e02f20>, 2339643.849722845)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81015b990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81010d850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3f2e350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2e039a0>, 2339644.438439851)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810150850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2e037e0>, 2339644.874959275)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3f2dcd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3f05cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2e00910>, 2339644.0919241)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3f05b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8101db010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81013c4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2e00210>, 2339644.232585399)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8101d9910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2e02740>, 2339644.944169538)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81013f310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8102c6290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2e01630>, 2339644.779764812)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8102c6090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3c16f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3dc1c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3f86690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2e00ad0>, 2339644.809071552)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3c14510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1a84360>, 2339643.060090537)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2e4bdd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1a86820>, 2339643.465546731)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3f85410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3e32d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2f33d20>, 2339647.08956513)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3e9bd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3e33d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2f307c0>, 2339645.549160807)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810273650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8103953d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2e01d30>, 2339645.678971047)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810394f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8101796d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810218410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2745780>, 2339645.482994816)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3e21b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2f30830>, 2339645.690065508)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8102191d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8103ebed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8102a4a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2f32900>, 2339645.614342369)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8102e2650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2f32350>, 2339645.580386382)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8102a4990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8103c3d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2f33ee0>, 2339645.989614056)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8103c3f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810309f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d38090f0>, 2339645.789602692)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81030a6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3e5b650>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2dbdc50>, 2339646.142844485)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3e5b790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8101c7490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3809a90>, 2339646.153258136)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c88b101f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8104dffd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810489450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8102984d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3f85810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2f337e0>, 2339646.839587722)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3f84b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2e032a0>, 2339646.513247106)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81048a150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d380ac10>, 2339646.688277374)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810345d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2f33000>, 2339644.892264361)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81029a650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81054f510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d380b700>, 2339647.346519816)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81054f910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8105d5a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81057eb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3808b40>, 2339647.854009361)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8105d7d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d380be00>, 2339647.158330071)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81057f0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8105cda10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81044bc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d380bc40>, 2339646.896781018)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8103b3e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d38083d0>, 2339647.274110048)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81044a550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810495c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d380a0b0>, 2339647.166860729)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810494210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81050bd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810651290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3500130>, 2339648.505219583)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810508fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3503850>, 2339648.445509414)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810652810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81067bfd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2f334d0>, 2339648.55317098)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81067ac90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810671150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8107b53d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3501080>, 2339648.477873249)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810673f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8107c2a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810693d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810336ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8106a7dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810698350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3502ba0>, 2339648.980428284)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8107b6ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d38f6040>, 2339647.943542007)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81060ad50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ef19b0>, 2339649.412022504)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810690290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3501320>, 2339648.643810353)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810336550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c88bbe4280>, 2339650.232335625)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810363950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3502b30>, 2339649.081458535)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810699e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8108d26d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d35006e0>, 2339649.73811349)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8108d2a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8108964d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d38f62e0>, 2339649.007823394)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810896110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8108a42d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ef3690>, 2339649.052203632)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8108b7d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8107ca150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8108c9e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2f30b40>, 2339649.813059681)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d373ae10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d38f5710>, 2339648.974564563)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81081d850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81086ed50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8108004d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d38f5550>, 2339649.297797497)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3ecad50>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2c9ac80>, 2339649.356324994)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810800050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8107f6dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d38f46e0>, 2339649.688670191)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8107f7890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810daaad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a17e00>, 2339650.678801196)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810da8710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810d9b9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a14910>, 2339650.258814541)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810d99910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810d39e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d38f72a0>, 2339650.296215054)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810d39f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810dac710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a16ac0>, 2339650.819784812)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810dae690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810d61fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810d1d950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810727f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d380b1c0>, 2339648.71062436)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8107263d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a15240>, 2339651.024254841)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810d617d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d38f66d0>, 2339650.631103175)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810d1ce90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810ede5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3502cf0>, 2339651.175849914)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810edc7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810ec9410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a14d00>, 2339651.04743967)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810ec9390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810e7d490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810264a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3c2af20>, 2339652.374611602)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8104089d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81086dad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3c28e50>, 2339651.661427114)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3ffa150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8115ec4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810e5e4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3c2b690>, 2339650.80859061)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8115ec450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3c29c50>, 2339651.018645679)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810e5dd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c88b0dcbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3c28bb0>, 2339651.054353149)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c88b0dc910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c88b1302d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810f1d050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3c28210>, 2339652.557424169)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c88b130510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a167b0>, 2339651.450809754)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810f1da10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c88c0e73d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3c2b1c0>, 2339652.130308629)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c88c0e6410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81067b1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c88b0bed90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3c2baf0>, 2339652.424682849)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81067bf50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3c289f0>, 2339652.628329976)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81074cb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810d16590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810f97e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d38f5630>, 2339651.48534194)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810e7d310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a17ee0>, 2339652.405408)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3bdb210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3b2a200>, 2339652.444819165)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810f94490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810d27910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3b2bd20>, 2339653.057212744)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8110e0c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c88bf84350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8110414d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3c284b0>, 2339653.043260632)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8110acfd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3b2b460>, 2339653.847344733)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8110da090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811033390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3b2b540>, 2339653.383309682)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81067b9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81102a650>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8111df150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e512b0>, 2339653.858269322)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81102a3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3b28ad0>, 2339653.325636158)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8111dda50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810ec3ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3b29940>, 2339653.29144892)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810f26e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8111bc990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811135010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3b2a4a0>, 2339653.718288573)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8105eb190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8110f0d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3d446e0>, 2339653.800942295)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8110f2550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81115e5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a172a0>, 2339654.376365047)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81115f150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c88b0bc590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c88b0bc6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8110d8550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3b28670>, 2339654.334422886)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811134490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a149f0>, 2339652.847434804)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c88b0bcdd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d35001a0>, 2339654.482975085)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811276c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3c2bd90>, 2339652.677053382)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810422090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810ec13d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810cff350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3d47e70>, 2339655.018572413)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810ece510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3d46900>, 2339654.682088981)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8112da9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811251fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3d44750>, 2339654.596933142)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811258890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8112b6a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8113caad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d38f4bb0>, 2339654.888849058)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8112b6790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3d47d90>, 2339655.378122643)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8113cbd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81139b050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3501630>, 2339655.455125222)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81139c4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81135e2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3b28a60>, 2339656.202904076)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810ddd850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81133ca10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3b280c0>, 2339655.51686124)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81133fd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8116cd010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3b29550>, 2339655.536861161)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8116cdcd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8111a7d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3e498d0>, 2339655.982823888)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811385850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811029f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3e487c0>, 2339656.133725438)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810f1d6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810ff7e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81163ea50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3e4b540>, 2339655.929530242)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8113e9bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3e48fa0>, 2339656.421368534)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81163f2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8116033d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3e48130>, 2339656.196113249)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811603a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811623810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811116f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3e4ac80>, 2339656.147282761)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811620d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3d44c20>, 2339654.281448464)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811116ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811318cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3e49e10>, 2339656.672376728)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81131a590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8116a9990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3f49860>, 2339656.741503118)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810f82910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811022c90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3f4a5f0>, 2339656.820693802)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811020310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811755650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811657a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3b29ef0>, 2339656.864426002)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811776910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3f48ec0>, 2339656.599146099)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81178fd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811645450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3f4b850>, 2339656.624948715)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811645cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811798850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8118e7890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3c2b380>, 2339657.582596273)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811798510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3f4a9e0>, 2339657.624579307)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8118e59d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8115f8f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3c28750>, 2339658.186263064)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8115f9210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81184b350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81176c990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81025ef20>, 2339657.562046688)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811366ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8113bee90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81181fa50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81025e7b0>, 2339657.820950079)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811748290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81025de10>, 2339657.589890209)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81181c290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8113ea090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8119814d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3e4b0e0>, 2339660.167031542)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8113e9e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d35023c0>, 2339658.599473676)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811981a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8118b0ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3f48670>, 2339657.94323263)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81116c4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81025f700>, 2339658.443845933)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8118b0a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8119dfb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81187ae10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81025f540>, 2339658.600537631)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811809790>\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8119166d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81025d240>, 2339658.915276056)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81191c4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8119ea790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81025f690>, 2339659.087759425)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8119e8050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811ab6390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3d46eb0>, 2339658.968697145)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811ab5950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811116cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811a65950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8104581a0>, 2339659.170032329)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8116467d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81045b930>, 2339658.956015721)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811916350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811a5e390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810458bb0>, 2339658.875661894)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811a5d9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8116a9650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811dd5290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3b28210>, 2339659.767461407)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8116a9b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3c29080>, 2339659.764269663)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811dd5510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81176c3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81045b7e0>, 2339660.217503426)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81176c690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811937c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811d5a110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d38f4590>, 2339662.049015861)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811d59e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811d56890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8104588a0>, 2339659.915146945)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811a8c4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811d74ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811ee8f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81025e6d0>, 2339658.275118732)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81187a210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81045a200>, 2339660.090532482)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811d693d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810459320>, 2339660.272456269)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811d76250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8106740c0>, 2339659.901670089)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811ee8510>\n",
      "Evaluating workflow:  33%|███▎      | 980/3000 [03:11<04:58,  6.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 985/3000 [03:11<02:59, 11.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 988/3000 [03:12<04:40,  7.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 992/3000 [03:12<04:16,  7.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 994/3000 [03:12<03:38,  9.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 996/3000 [03:13<04:55,  6.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 999/3000 [03:13<05:09,  6.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 1003/3000 [03:14<04:35,  7.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 1004/3000 [03:14<04:30,  7.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▎      | 1008/3000 [03:15<04:08,  8.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▎      | 1011/3000 [03:15<03:04, 10.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1013/3000 [03:15<02:55, 11.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1015/3000 [03:15<03:44,  8.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1017/3000 [03:15<03:55,  8.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1019/3000 [03:16<04:37,  7.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1020/3000 [03:16<04:59,  6.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 1023/3000 [03:16<04:46,  6.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 1028/3000 [03:17<02:47, 11.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 1033/3000 [03:17<03:19,  9.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1035/3000 [03:18<03:54,  8.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 1037/3000 [03:18<04:23,  7.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▍      | 1039/3000 [03:19<05:44,  5.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 1041/3000 [03:19<04:24,  7.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 1043/3000 [03:19<03:48,  8.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▍      | 1048/3000 [03:19<02:57, 11.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 1050/3000 [03:20<03:23,  9.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 1053/3000 [03:20<03:09, 10.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 1055/3000 [03:20<04:02,  8.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 1058/3000 [03:20<03:48,  8.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▌      | 1060/3000 [03:21<04:49,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 1062/3000 [03:21<03:56,  8.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 1066/3000 [03:21<03:23,  9.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1069/3000 [03:22<02:41, 11.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1071/3000 [03:22<03:44,  8.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1075/3000 [03:23<04:18,  7.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 1078/3000 [03:23<04:46,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1081/3000 [03:23<03:23,  9.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1083/3000 [03:24<04:10,  7.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1085/3000 [03:24<03:40,  8.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1087/3000 [03:24<04:26,  7.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▋      | 1088/3000 [03:25<06:12,  5.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▋      | 1092/3000 [03:25<04:42,  6.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▋      | 1095/3000 [03:25<03:16,  9.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 1099/3000 [03:26<03:22,  9.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1103/3000 [03:26<02:55, 10.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 1107/3000 [03:27<03:27,  9.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1109/3000 [03:27<05:01,  6.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1112/3000 [03:27<03:58,  7.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 1116/3000 [03:28<03:36,  8.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1118/3000 [03:28<03:47,  8.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1121/3000 [03:28<03:12,  9.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1123/3000 [03:29<03:08,  9.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1125/3000 [03:29<03:20,  9.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1127/3000 [03:29<04:58,  6.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1128/3000 [03:30<05:33,  5.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1131/3000 [03:30<04:43,  6.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1135/3000 [03:30<02:45, 11.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1139/3000 [03:31<02:58, 10.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1144/3000 [03:31<02:04, 14.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1146/3000 [03:31<03:12,  9.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1148/3000 [03:32<05:59,  5.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1154/3000 [03:33<03:37,  8.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▊      | 1159/3000 [03:33<02:40, 11.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▊      | 1161/3000 [03:33<02:51, 10.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1164/3000 [03:33<02:22, 12.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1166/3000 [03:34<03:00, 10.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1168/3000 [03:34<03:41,  8.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 1173/3000 [03:47<36:38,  1.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:46:31.256\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are assisting with the interpretation of perturbation-based expression data. In K562 cells, SOCS1 is perturbed and ZNF280B expression is quantified. Does this perturbation result in a significant change in ZNF280B expression compared with control cells? Respond exactly as 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are assisting with the interpretation of perturbation-based expression data. In K562 cells, SOCS1 is perturbed and ZNF280B expression is quantified. Does this perturbation result in a significant change in ZNF280B expression compared with control cells? Respond exactly as 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 1102}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 3 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:46:31.291\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a functional genomics specialist. In K562 cells, we perturb SYVN1 and monitor EPB42 expression. Decide whether this perturbation leads to a significant alteration in EPB42 expression. Answer only in the format 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are a functional genomics specialist. In K562 cells, we perturb SYVN1 and monitor EPB42 expression. Decide whether this perturbation leads to a significant alteration in EPB42 expression. Answer only in the format 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 1718}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 3 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1175/3000 [03:57<1:05:26,  2.15s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:46:31.857\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1176/3000 [03:57<57:52,  1.90s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:46:32.405\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1177/3000 [03:58<50:19,  1.66s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:46:32.437\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:46:33.998\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to BHLHE40 and then measure expression of NRIP1. Does this perturbation cause a significant change in NRIP1 expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to BHLHE40 and then measure expression of NRIP1. Does this perturbation cause a significant change in NRIP1 expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 124}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 1182/3000 [04:00<22:48,  1.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2026-01-13 12:46:34.428\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:46:34.472\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to MRGBP and then measure expression of WHSC1. Does this perturbation cause a significant change in WHSC1 expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to MRGBP and then measure expression of WHSC1. Does this perturbation cause a significant change in WHSC1 expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 1484}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1184/3000 [04:00<15:38,  1.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:46:34.541\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:46:34.575\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:46:34.578\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are assisting with the interpretation of perturbation-based expression data. In K562 cells, MTHFD1 is perturbed and RPL39 expression is quantified. Does this perturbation result in a significant change in RPL39 expression compared with control cells? Respond exactly as 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are assisting with the interpretation of perturbation-based expression data. In K562 cells, MTHFD1 is perturbed and RPL39 expression is quantified. Does this perturbation result in a significant change in RPL39 expression compared with control cells? Respond exactly as 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 1742}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 1186/3000 [04:00<11:02,  2.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:46:34.591\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to DHDDS and then measure expression of HM13. Does this perturbation cause a significant change in HM13 expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to DHDDS and then measure expression of HM13. Does this perturbation cause a significant change in HM13 expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 4514}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:46:34.611\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:46:34.613\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert analyst of perturbation datasets. For K562 cells subjected to perturbation of ATP5B, does the expression profile of SERPINH1 indicate a significant change relative to control conditions? Reply strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are an expert analyst of perturbation datasets. For K562 cells subjected to perturbation of ATP5B, does the expression profile of SERPINH1 indicate a significant change relative to control conditions? Reply strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 1647}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:46:34.624\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to HARS and then measure expression of VPS41. Does this perturbation cause a significant change in VPS41 expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to HARS and then measure expression of VPS41. Does this perturbation cause a significant change in VPS41 expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 634}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:46:34.636\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a genomics expert evaluating perturbation experiments. In K562 cells, SOCS1 is perturbed and the expression of ZFHX3 is measured. Determine whether ZFHX3 shows a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are a genomics expert evaluating perturbation experiments. In K562 cells, SOCS1 is perturbed and the expression of ZFHX3 is measured. Determine whether ZFHX3 shows a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 1136}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:46:34.661\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a functional genomics specialist. In K562 cells, we perturb CAD and monitor CTD-2587M2.1 expression. Decide whether this perturbation leads to a significant alteration in CTD-2587M2.1 expression. Answer only in the format 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are a functional genomics specialist. In K562 cells, we perturb CAD and monitor CTD-2587M2.1 expression. Decide whether this perturbation leads to a significant alteration in CTD-2587M2.1 expression. Answer only in the format 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 1918}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 1194/3000 [04:01<06:45,  4.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 1196/3000 [04:03<09:36,  3.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|████      | 1204/3000 [04:03<04:39,  6.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1209/3000 [04:03<03:15,  9.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1212/3000 [04:04<03:10,  9.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1215/3000 [04:05<06:42,  4.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1219/3000 [04:06<05:57,  4.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1224/3000 [04:06<03:53,  7.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1228/3000 [04:07<03:03,  9.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1230/3000 [04:07<03:24,  8.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1232/3000 [04:07<03:31,  8.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1234/3000 [04:08<03:32,  8.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1236/3000 [04:08<04:31,  6.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████▏     | 1241/3000 [04:08<03:11,  9.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████▏     | 1244/3000 [04:09<02:38, 11.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1246/3000 [04:09<03:13,  9.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1251/3000 [04:09<02:46, 10.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1253/3000 [04:10<03:20,  8.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1254/3000 [04:10<04:23,  6.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1258/3000 [04:10<03:17,  8.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1260/3000 [04:11<03:24,  8.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1264/3000 [04:11<02:43, 10.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1266/3000 [04:11<02:33, 11.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1268/3000 [04:11<02:49, 10.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1270/3000 [04:12<03:06,  9.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1273/3000 [04:12<05:07,  5.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1279/3000 [04:13<02:31, 11.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1281/3000 [04:13<02:57,  9.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1283/3000 [04:14<04:12,  6.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1285/3000 [04:14<05:03,  5.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1287/3000 [04:14<04:26,  6.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1293/3000 [04:15<03:02,  9.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1295/3000 [04:15<02:55,  9.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1299/3000 [04:16<04:32,  6.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1303/3000 [04:16<03:21,  8.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▎     | 1308/3000 [04:17<02:48, 10.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▎     | 1312/3000 [04:17<02:28, 11.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1315/3000 [04:18<03:51,  7.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1319/3000 [04:18<03:27,  8.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1321/3000 [04:18<03:18,  8.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1324/3000 [04:19<03:28,  8.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1326/3000 [04:19<04:16,  6.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1330/3000 [04:19<03:05,  9.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1332/3000 [04:20<04:01,  6.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▍     | 1336/3000 [04:20<03:00,  9.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▍     | 1339/3000 [04:21<03:33,  7.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▍     | 1343/3000 [04:21<02:51,  9.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1345/3000 [04:21<02:26, 11.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1347/3000 [04:22<03:36,  7.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1350/3000 [04:22<02:42, 10.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1352/3000 [04:22<02:58,  9.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1354/3000 [04:22<02:56,  9.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▌     | 1358/3000 [04:23<03:06,  8.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1360/3000 [04:23<03:05,  8.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1363/3000 [04:23<03:20,  8.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1365/3000 [04:24<04:38,  5.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1371/3000 [04:24<03:05,  8.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1373/3000 [04:25<04:30,  6.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1377/3000 [04:25<03:33,  7.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1381/3000 [04:26<03:29,  7.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1384/3000 [04:26<03:25,  7.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1387/3000 [04:26<03:04,  8.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▋     | 1389/3000 [04:27<03:31,  7.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▋     | 1393/3000 [04:27<03:16,  8.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1396/3000 [04:28<03:13,  8.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1401/3000 [04:28<02:36, 10.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1407/3000 [04:29<02:36, 10.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1409/3000 [04:29<02:25, 10.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1414/3000 [04:30<03:43,  7.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1416/3000 [04:30<03:52,  6.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1422/3000 [04:31<02:27, 10.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1426/3000 [04:31<02:27, 10.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1428/3000 [04:31<02:44,  9.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1432/3000 [04:32<03:42,  7.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1435/3000 [04:32<02:40,  9.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1437/3000 [04:33<03:32,  7.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1439/3000 [04:33<03:07,  8.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1441/3000 [04:33<03:32,  7.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1443/3000 [04:33<03:13,  8.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1445/3000 [04:34<03:30,  7.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1448/3000 [04:34<03:41,  7.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1454/3000 [04:35<03:31,  7.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1455/3000 [04:37<07:56,  3.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▊     | 1456/3000 [04:38<10:04,  2.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:47:21.683\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in gene expression analysis. For K562 cells, assess whether perturbation of CARS is associated with a significant change in NAV2 expression compared with unperturbed controls. Answer strictly as 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are an expert in gene expression analysis. For K562 cells, assess whether perturbation of CARS is associated with a significant change in NAV2 expression compared with unperturbed controls. Answer strictly as 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 4865}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 13 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▊     | 1457/3000 [04:47<49:03,  1.91s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:47:21.769\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 13 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:47:21.839\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 13 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▊     | 1459/3000 [04:47<34:02,  1.33s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:47:21.850\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 10 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:47:21.956\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 12 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▊     | 1461/3000 [04:48<23:50,  1.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:47:22.229\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 12 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1463/3000 [04:48<17:32,  1.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:47:22.258\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a genomics expert evaluating perturbation experiments. In K562 cells, ASCC3 is perturbed and the expression of SKIL is measured. Determine whether SKIL shows a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are a genomics expert evaluating perturbation experiments. In K562 cells, ASCC3 is perturbed and the expression of SKIL is measured. Determine whether SKIL shows a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 2946}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 12 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:47:22.268\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a genomics expert evaluating perturbation experiments. In K562 cells, DHDDS is perturbed and the expression of CLDN11 is measured. Determine whether CLDN11 shows a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are a genomics expert evaluating perturbation experiments. In K562 cells, DHDDS is perturbed and the expression of CLDN11 is measured. Determine whether CLDN11 shows a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 4546}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 12 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:47:22.304\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a functional genomics specialist. In K562 cells, we perturb DNAJC19 and monitor HIST1H2BC expression. Decide whether this perturbation leads to a significant alteration in HIST1H2BC expression. Answer only in the format 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are a functional genomics specialist. In K562 cells, we perturb DNAJC19 and monitor HIST1H2BC expression. Decide whether this perturbation leads to a significant alteration in HIST1H2BC expression. Answer only in the format 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 4778}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 12 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1467/3000 [04:48<09:56,  2.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:47:33.949\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a domain expert in functional genomics. For experiments carried out in K562 cells, we perturb SYVN1 and examine the expression of USP37. Does perturbing SYVN1 lead to a significant change in USP37 expression? Reply only with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are a domain expert in functional genomics. For experiments carried out in K562 cells, we perturb SYVN1 and examine the expression of USP37. Does perturbing SYVN1 lead to a significant change in USP37 expression? Reply only with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 1731}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:47:34.027\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert analyst of perturbation datasets. For K562 cells subjected to perturbation of SLC39A7, does the expression profile of PELO indicate a significant change relative to control conditions? Reply strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are an expert analyst of perturbation datasets. For K562 cells subjected to perturbation of SLC39A7, does the expression profile of PELO indicate a significant change relative to control conditions? Reply strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 2117}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1469/3000 [05:00<44:30,  1.74s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:47:34.048\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are assisting with the interpretation of perturbation-based expression data. In K562 cells, TIMM23 is perturbed and RP11-138C9.1 expression is quantified. Does this perturbation result in a significant change in RP11-138C9.1 expression compared with control cells? Respond exactly as 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are assisting with the interpretation of perturbation-based expression data. In K562 cells, TIMM23 is perturbed and RP11-138C9.1 expression is quantified. Does this perturbation result in a significant change in RP11-138C9.1 expression compared with control cells? Respond exactly as 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 832}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:47:34.233\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are assisting with the interpretation of perturbation-based expression data. In K562 cells, HYOU1 is perturbed and TOPBP1 expression is quantified. Does this perturbation result in a significant change in TOPBP1 expression compared with control cells? Respond exactly as 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are assisting with the interpretation of perturbation-based expression data. In K562 cells, HYOU1 is perturbed and TOPBP1 expression is quantified. Does this perturbation result in a significant change in TOPBP1 expression compared with control cells? Respond exactly as 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 3802}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1471/3000 [05:00<33:24,  1.31s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:47:34.741\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in single-cell biology and functional genomics. In K562 cells, DNAJC19 is perturbed and the expression of LST1 is measured. Does this perturbation cause a significant change in LST1 expression? Answer strictly in the format 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are an expert in single-cell biology and functional genomics. In K562 cells, DNAJC19 is perturbed and the expression of LST1 is measured. Does this perturbation cause a significant change in LST1 expression? Answer strictly in the format 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 4750}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1473/3000 [05:00<26:00,  1.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:47:34.808\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to MARS and then measure expression of DUSP21. Does this perturbation cause a significant change in DUSP21 expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to MARS and then measure expression of DUSP21. Does this perturbation cause a significant change in DUSP21 expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 2744}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:47:34.824\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a domain expert in functional genomics. For experiments carried out in K562 cells, we perturb AMIGO3 and examine the expression of ESCO1. Does perturbing AMIGO3 lead to a significant change in ESCO1 expression? Reply only with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are a domain expert in functional genomics. For experiments carried out in K562 cells, we perturb AMIGO3 and examine the expression of ESCO1. Does perturbing AMIGO3 lead to a significant change in ESCO1 expression? Reply only with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 861}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:47:34.854\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1476/3000 [05:00<16:42,  1.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:47:34.898\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a domain expert in functional genomics. For experiments carried out in K562 cells, we perturb SLC39A7 and examine the expression of PELO. Does perturbing SLC39A7 lead to a significant change in PELO expression? Reply only with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are a domain expert in functional genomics. For experiments carried out in K562 cells, we perturb SLC39A7 and examine the expression of PELO. Does perturbing SLC39A7 lead to a significant change in PELO expression? Reply only with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 2111}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:47:34.909\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:47:35.179\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1479/3000 [05:01<11:51,  2.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:47:35.448\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:47:35.573\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1481/3000 [05:01<10:09,  2.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:47:36.194\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811d455d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81025f000>, 2339660.858208577)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811e86ed0>\n",
      "Evaluating workflow:  49%|████▉     | 1483/3000 [05:02<11:11,  2.26it/s]Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811ec9350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810676270>, 2339660.881096751)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811ec8710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811ad5350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811e4fed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3f4aac0>, 2339661.355099752)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811de4c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810677540>, 2339660.952674992)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811377d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81188af50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810745f60>, 2339661.649736464)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8124574d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d174a290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810d77930>, 2339663.095333457)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d177bdd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d114d250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810746350>, 2339663.542388425)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c88c1f2e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811d54810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8106741a0>, 2339665.195549885)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811d56bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0eb2590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810ec57f0>, 2339665.832848375)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0eb3e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0d2d510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8111ec910>, 2339665.545933658)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0d2d410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0d466d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8111ef850>, 2339666.750793139)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0d45e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d11ef390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8111ec4b0>, 2339666.450262393)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d11ee950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0e2c690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811205780>, 2339666.254185886)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0e2c250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c88c22b750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811205240>, 2339666.668198788)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0dfb9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0e8f9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811205710>, 2339668.316579546)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0c4e910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0ba6f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81162be70>, 2339669.494689633)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0ba7290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0a3ee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81172a820>, 2339683.797329811)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0a3fa10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0a4da90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81172b8c0>, 2339694.532234448)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0a4e8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0cd3610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81172a7b0>, 2339680.174609932)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0bc8c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d09753d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81182c670>, 2339684.280499875)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0975910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d099c910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811939550>, 2339683.603407622)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d099c890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d091a6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0986cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d09a4e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d09cd950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d09f4990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d08014d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810ec5080>, 2339680.164584993)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0919850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8111ed400>, 2339680.283518264)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0986990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81162ac80>, 2339684.231805698)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d09a61d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81182c830>, 2339693.79069919)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d09cd750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811629a90>, 2339683.528765356)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d09f5410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8113150f0>, 2339683.624620049)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0801390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0aec190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811729b00>, 2339684.186227393)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0aacc50>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d08c26d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811a3f150>, 2339698.054243912)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d08c2950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0754710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8124ce430>, 2339696.794032659)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0754310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d077cdd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d39a90>, 2339700.096083862)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d077d2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0898210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d089e250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811a3cde0>, 2339696.764525576)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d089e050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d070bc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d072ec10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0747910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d07835d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811a3dc50>, 2339694.306487916)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d08987d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811a3dfd0>, 2339699.996459329)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0709950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8124cde80>, 2339696.749290657)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d072ead0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8124cd940>, 2339696.770783099)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d09f6bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d3aac0>, 2339699.753612693)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0783450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d082eb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d39400>, 2339699.423560105)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d082e750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0662fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c812517e70>, 2339699.329631946)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0663810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d07e2790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810833700>, 2339699.764897734)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d068ffd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d06ba6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8108311d0>, 2339699.271627193)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0688a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d086a6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0605350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d06101d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d08699d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81182c2f0>, 2339699.595724176)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0869a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d3bd20>, 2339701.483595072)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d06051d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d3b700>, 2339701.299633564)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0610690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c812517000>, 2339699.615625549)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d063ae10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d052cc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81249f380>, 2339699.485426065)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0631550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0571310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81249edd0>, 2339700.052407204)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0571150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d06af110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811a3c590>, 2339701.692526283)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d05de850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d068ad90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d053ee50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d05542d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d05b18d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81249e430>, 2339701.237142288)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d068aa10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81249fc40>, 2339699.909982891)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d054ba10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81249f5b0>, 2339699.945753374)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d053f750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81249d080>, 2339699.709865038)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d05b1750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d042e450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0404490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8108324a0>, 2339701.860114206)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d042e4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810832a50>, 2339701.717960072)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d04048d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0317b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d188c750>, 2339703.664376905)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0317a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0334590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811a3f380>, 2339703.282492936)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0334910>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0509fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d04905d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d049fe50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d04d7310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d04e3850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d04e9090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0548ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81249c440>, 2339703.168213935)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d04d71d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d188ee40>, 2339703.535968708)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d04e3650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110f82f0>, 2339701.392858247)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0548b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d188e6d0>, 2339703.349172207)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d04ea9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d3ab30>, 2339701.474996938)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d049fd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81249d240>, 2339701.709367913)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0960990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81193ac80>, 2339702.005806426)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d04903d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d05e26d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811a3e2e0>, 2339703.545004469)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d05e2990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0247610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d034b700>, 2339704.912312899)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d047a090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d030b2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d044da90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d03cea10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d07e0f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0348520>, 2339703.599726535)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d03cf0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110fba10>, 2339703.682133261)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d07e0b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d188f000>, 2339703.626151289)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d030bd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d188c670>, 2339703.771118965)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d044eb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d02d0050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0273000>, 2339706.530317577)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d02d0910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d01c1fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0181a90>, 2339706.608118242)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d01c2310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0363750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d02d98d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d02f23d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0180ec0>, 2339707.305436923)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0363610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110f9710>, 2339705.637085461)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d02da010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0272740>, 2339705.540727243)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d02f0e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d01c6b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0348d00>, 2339707.700897207)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d01c7250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7f1aed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0182120>, 2339707.89471441)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0245850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7f53c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7f62c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0272270>, 2339707.69089641)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d08e7410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811314590>, 2339707.85245502)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7f632d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d020fa10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7f890d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c7f74600>, 2339707.949380763)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d020f550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d188f8c0>, 2339707.947659036)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7f897d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7fa2750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7fb0950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0272cf0>, 2339707.980841897)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7fa2c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d01834d0>, 2339707.998163987)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7fb0810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7fc07d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c7f75630>, 2339707.911428936)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7fc0b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7fee890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c7f75fd0>, 2339709.210294645)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7feed90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7e12d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d02703d0>, 2339709.247298682)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7e12b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7fd97d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7e47b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0349cc0>, 2339710.16374958)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7e33350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c7f778c0>, 2339708.759314658)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7e47a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7e53290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0273cb0>, 2339709.315721137)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7e53090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d018ecd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7e6d050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c7e800c0>, 2339709.091967487)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d018ee10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d188ff50>, 2339709.019776293)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7e6d710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7e32690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7ea47d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8118fec90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8124dc4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811e25dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c88c2fe610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8124cb610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c88c002a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c812476c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811798f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c88bfbf6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8124377d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811d68c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1778d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1782f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811ad5690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d179b9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d114f950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811d57190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d11a5d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d116dc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d11c9350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d11d63d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d11edb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0e1cd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0e3c4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0e50d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0e75310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0e66590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0e8dbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0e8c650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0ed3ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0ef6ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0d14190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0e1d210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d11df110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0d226d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0d5a210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0d944d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0daff50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0dc28d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0de8910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0c168d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0c333d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0c4c710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0d90890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0dfa6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0c7d5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0c26dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0ca5390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0cadf10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0cf5b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0d94c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0b2d250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0cd6850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0b40390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0b4a790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0b5d150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0b76350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0ef4e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0d2cc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0ba5590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0bc38d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0beec50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0a18290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0b42dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0a80cd0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0cb7d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0a3c590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0cb7650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0ad9690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0be2550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0be0150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0a4f550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0ba5d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0957e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d094ec90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d068a090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d06f5dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0509c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d044de10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0463390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d04af790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d036e910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d04c5750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0384550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d03dc4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d03ea6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d037a350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0585250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d035cad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d022df50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0245210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d02819d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0256390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0257450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d02e0ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0116b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0120a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0143550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d01672d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0173490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d019f310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d01cd210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d02acbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d017d6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7f0cc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c7e80520>, 2339708.912992122)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7e31e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81249d550>, 2339709.608913126)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7ea4c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810676970>, 2339660.914071424)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811602a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810676cf0>, 2339661.150104443)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8117a73d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810676a50>, 2339661.411795761)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811e24850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810674c20>, 2339661.570967081)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c88c2fd110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810676580>, 2339661.376474707)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8124ca790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81045a4a0>, 2339661.491053266)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c88c000d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810746ba0>, 2339662.058798306)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c812474a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810747930>, 2339662.124798877)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811de7e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810747700>, 2339662.179241337)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c88bfbe750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810744c20>, 2339662.709855376)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c812435cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3f4be70>, 2339664.848535253)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c812435750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8107467b0>, 2339662.647305071)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1c8d150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3f4bc40>, 2339662.563297846)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1780750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810675cc0>, 2339663.136184088)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811ad5c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810674de0>, 2339663.271345485)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d179b2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810d765f0>, 2339663.126386311)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d114f810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810d76350>, 2339663.637380749)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811d56d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810458ec0>, 2339663.427324203)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d11a5b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810d75160>, 2339663.149595923)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d116d2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810d75780>, 2339663.604694859)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d11c9150>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81045a190>, 2339663.980656641)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d11d6b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c88c4537e0>, 2339664.050468974)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d11ee210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c88c4535b0>, 2339664.405185427)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8124b3210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c88c4533f0>, 2339664.034519736)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0e3da10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810ec6820>, 2339664.410559274)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0e51410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810ec6430>, 2339664.908151006)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0e751d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810ec77e0>, 2339664.923239869)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0e65690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810745c50>, 2339665.282714529)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0e8e210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810459a90>, 2339665.237305502)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0ebbf90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810ec5a90>, 2339665.608693087)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0ed3dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810745cc0>, 2339665.530669266)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0eaf550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8111ef0e0>, 2339665.727403569)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0d0fdd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81025c440>, 2339664.308521722)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0e1db10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c88c453f50>, 2339663.916186006)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0e2ce10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810d77e00>, 2339665.738886386)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0d23010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8111ec130>, 2339666.484264083)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0d59f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8111ed6a0>, 2339666.314343027)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811edca50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8111ee040>, 2339666.758936708)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0dafe50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8112055c0>, 2339668.492266229)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0dc0710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8112070e0>, 2339666.578924307)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0de8610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811204670>, 2339667.231963527)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0c17010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810d771c0>, 2339667.188498776)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0dfae10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811204de0>, 2339667.12193937)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1784750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8111eeba0>, 2339666.402669982)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0d90750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811207d20>, 2339668.41249168)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0c7ae10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811206ba0>, 2339667.893573399)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0d2c4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8112072a0>, 2339667.887069078)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0c26ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811317540>, 2339667.754320965)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0ca58d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811316ba0>, 2339667.897724164)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0cad9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8111ed0f0>, 2339668.487636045)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0cf5a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811317a10>, 2339668.578102872)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0d946d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811316f20>, 2339668.343554865)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0b2d910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8111ec6e0>, 2339668.632649102)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0cc77d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8111ed080>, 2339668.466412239)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0b41ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811315a20>, 2339669.024938239)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0b4ad50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8111ee890>, 2339668.925894075)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0b5cc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811205e10>, 2339669.384345385)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0b76150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811316820>, 2339669.04528212)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0ef4c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811317f50>, 2339668.483373197)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0c20950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81162bb60>, 2339683.834183835)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0b9a6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811628a60>, 2339669.4628523)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0bc36d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811628de0>, 2339670.100699882)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0e67250>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811629320>, 2339669.978853206)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0a13b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811728210>, 2339670.161417231)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0b42f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81162a0b0>, 2339693.670442966)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0b99250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810ec7540>, 2339693.672411533)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0cb64d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811728d70>, 2339670.451486356)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0dc2610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811729da0>, 2339670.393480594)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0dc26d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811315e80>, 2339694.0932524)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0ad9490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8112074d0>, 2339670.069576933)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0be2350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81162bd90>, 2339670.906928061)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0bd7690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811204980>, 2339670.476476354)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0a4f390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81162aba0>, 2339694.5454979)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0ba5c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81182fd90>, 2339694.054156159)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0957d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81182e2e0>, 2339683.633421251)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d094c4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81182ea50>, 2339696.780929271)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d068a850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8108330e0>, 2339701.417192138)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d06f4290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d3ad60>, 2339699.328572708)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d050a410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110facf0>, 2339701.721723176)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0ba6350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d38ad0>, 2339702.191008925)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0462f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81249cbb0>, 2339702.017602952)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d04af650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0349010>, 2339703.559512265)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d036efd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d38b40>, 2339702.610415793)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d04c5550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81249e2e0>, 2339703.466846918)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0386cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0349b70>, 2339703.981785833)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d03dd5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d034a0b0>, 2339703.84404788)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d03eb290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d034a270>, 2339704.580028774)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d037bb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81172a970>, 2339705.617396143)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d05d9010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d034add0>, 2339704.312764569)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d035d850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d034af90>, 2339703.91029476)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d022e410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d38440>, 2339704.455636593)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0245010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0270830>, 2339705.182126245)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d02817d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d034ba10>, 2339705.585422165)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0256f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d188ea50>, 2339705.084444428)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d03638d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0272510>, 2339706.106571156)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d02e2810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0272900>, 2339705.71606022)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0115c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0272dd0>, 2339705.785073288)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0120850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0273380>, 2339706.665864677)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0143190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810830fa0>, 2339706.43032051)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d01670d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81249f1c0>, 2339706.694274522)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d05dc4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d034a970>, 2339706.282144466)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d019fa10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d01818d0>, 2339706.841628745)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d01cd010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d390f0>, 2339705.756452998)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d02acf10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0180210>, 2339706.648452276)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d017d650>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110fb9a0>, 2339707.960385937)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7f0d150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811d9a490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d090c350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d08e4250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810744360>, 2339662.014894078)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81167c8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811315550>, 2339693.786767392)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d090cd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81193acf0>, 2339697.326712962)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d07b0990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7d34a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7e7fad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d3a200>, 2339711.019249804)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7d07190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d3b850>, 2339710.705092983)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7e7c710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7efbd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81182d8d0>, 2339710.325754031)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7efa450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0527350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81182d780>, 2339710.413707743)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0525290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7e50790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811939cc0>, 2339710.759919907)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7e50fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d066ef10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d084c990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811939320>, 2339710.857472332)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d066ced0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811a3f310>, 2339711.93810841)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d084e750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d076cc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811a3cb40>, 2339712.302363871)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d076ef50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0630e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c7f77620>, 2339712.375771046)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7edb190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d061af90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d07e3c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8113173f0>, 2339711.283557727)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d061a950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8112075b0>, 2339711.605255784)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d079b790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0830d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7ed94d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7f20fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0472bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7e32cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0402ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7fa35d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0272970>, 2339711.76164237)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7f20c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810833ee0>, 2339711.272712556)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7fa3710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8124cd240>, 2339711.55555037)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7d46750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0273a80>, 2339710.981836941)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0470a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81182ee40>, 2339711.466078688)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0402f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811206580>, 2339711.493028231)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0830390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7f63790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1fc5c50>, 2339709.82546196)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7ed9f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8108320b0>, 2339710.155972058)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7f62fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d03cf550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0682810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d06b7290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0349780>, 2339712.430656243)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0584fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0817790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d071d290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d06d92d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810830590>, 2339712.965520616)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d071d0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a158d0>, 2339713.37769254)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d06da550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d06b6490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d08faed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a14210>, 2339713.243801898)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0783bd0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a179a0>, 2339712.960880699)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d08fab90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d063bc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81193b0e0>, 2339713.612212422)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7f0d550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0783750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d063b1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a16f20>, 2339713.545294045)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0782810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a162e0>, 2339713.385638901)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d06acf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d063a090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d067d190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a15f60>, 2339714.051399507)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d08a8810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0bb0850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a16b30>, 2339713.644857936)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0bb0910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0982350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0890850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d38e50>, 2339712.758375522)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d03cc210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0757f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81193b7e0>, 2339712.111465802)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d06b6a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810833460>, 2339713.456233084)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0814990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a16270>, 2339714.045184641)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d067fb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811316e40>, 2339713.621974889)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0982f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e13770>, 2339713.508737289)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0890fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c7f742f0>, 2339711.640179311)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0754550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d082d0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811316270>, 2339717.127524825)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d082f3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d09711d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d02723c0>, 2339714.88748872)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0971410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0b6bf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0613a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8124cd9b0>, 2339714.211751523)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7ef8f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810ec5be0>, 2339714.544035176)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0610d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d076c4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0971d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810832890>, 2339714.523419525)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d076f410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c88bc02820>, 2339714.846298855)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d09b1890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0986350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0a26c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a17a10>, 2339715.569281587)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d068ee10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c812416ba0>, 2339715.732076231)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0a27850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0a8eb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a17e00>, 2339715.445746125)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0a8f190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0c726d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c812516dd0>, 2339715.516990941)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0c4d410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0e51950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c812417af0>, 2339715.280440989)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0e52ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0b82350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0e1f350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c812417d20>, 2339715.992605831)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0b822d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c812416b30>, 2339716.16004464)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1190e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c88bfc9b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c812416e40>, 2339716.014349038)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c88bfcb3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d11ee0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8125ea6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8124163c0>, 2339715.622949)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d11ee6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a165f0>, 2339715.826169489)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c9b23630d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d076d090>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c88bbe4910>, 2339715.809348222)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811f3c8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811d5a350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a14a60>, 2339716.453147912)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811d69e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d085ee90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81193ab30>, 2339716.382781872)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d076e0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811d9c250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a17690>, 2339716.106800813)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811d9f210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811ec0490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811a74250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810349e10>, 2339716.638447934)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811ec01d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81034ac10>, 2339716.425512839)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811a9c2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81178e550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811904410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810349b00>, 2339717.070459385)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d090c390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81034b9a0>, 2339717.03979664)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d072c050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811969a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81187ba50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8118592d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a14910>, 2339717.356753578)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811859090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0c98890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3602ac0>, 2339717.908093928)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0c9a590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8118d95d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811276f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c88bbe44b0>, 2339718.390450994)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c812410310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3602b30>, 2339718.894935364)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811276810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81161ef50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c812435510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8111efaf0>, 2339717.629663163)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8119699d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8124159b0>, 2339718.109648979)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81187a9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3601780>, 2339717.819311529)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81161e910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c88bbe5080>, 2339716.748617158)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c812434b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81163e710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81139b4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3601cc0>, 2339718.386249798)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811398050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811c61bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3602510>, 2339718.862174373)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811c63750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811386f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81249f070>, 2339718.909079455)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811384b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81120cc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8111f8e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d35e4bb0>, 2339718.867565518)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8111a44d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d35e6f20>, 2339718.463431517)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8111fa950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8111bced0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d35e6f90>, 2339718.30779436)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81121ee10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81129b690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a160b0>, 2339719.514197625)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811d5b990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8117c4bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d35e65f0>, 2339720.032093562)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8117c4790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81120b610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8110c3310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81172bc40>, 2339720.142754532)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811209450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811729fd0>, 2339719.495610405)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8110c0a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c88bc3e590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811629b00>, 2339718.932833056)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81102a950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8110adf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8116007d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3600f30>, 2339718.306592764)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81163c490>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d063b850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810d74910>, 2339720.046870171)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0e04ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3603850>, 2339717.954131248)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811601390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3600c20>, 2339718.022246364)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d063a9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c98beb24d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1a85a90>, 2339719.871830032)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8116004d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81104f050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c88b0dca50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810349940>, 2339720.671967578)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810de7c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c7f76b30>, 2339721.089707491)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c88b0dcd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8115d6890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1a87ee0>, 2339720.538267448)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8115d6250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81073d4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1a854e0>, 2339721.992578543)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810e5f910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81121e9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810f4fa10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1a85240>, 2339721.938388349)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81121cf90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1a86970>, 2339720.468828892)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810f4d550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d08c22d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c7f76740>, 2339720.105126378)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811ceb110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c812411050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1a87c40>, 2339720.687652924)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8115de510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810d5dd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d35e4210>, 2339722.214791864)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810d5d050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810d69a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8108c8d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3603310>, 2339722.043156362)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810d60b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ca1390>, 2339722.053673017)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8108c9010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810d26990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810801f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ca1be0>, 2339721.558097239)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810f4e550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ca2350>, 2339720.78052932)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810802750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810d99e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ca3770>, 2339722.038582858)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810dac690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81067a950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810685650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ca31c0>, 2339721.59811172)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81067ac90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ca0e50>, 2339722.151892016)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d08e7310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810421150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ca17f0>, 2339722.460839541)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810422190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8104626d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8108a4990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ca0520>, 2339722.945087966)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810460350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1a86120>, 2339722.633578592)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8108a5610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8106bb750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810558e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ca2580>, 2339722.502391136)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810fbb610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d30bf0e0>, 2339722.674064812)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81055b290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810337e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8104d1f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d30bf700>, 2339722.920031358)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8104d07d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81061af50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810271990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d30be9e0>, 2339723.492895057)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81061b290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1a840c0>, 2339723.333810368)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810f6b0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c88b0ec310>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ae1e80>, 2339722.617102542)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c88b0c4090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810635010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d36014e0>, 2339723.47961081)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810635a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810f25e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81040f490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ae1cc0>, 2339724.080764662)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810cff190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ae0f30>, 2339724.287481667)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8105644d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3600280>, 2339722.615996921)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810337f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3f04e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d30bda20>, 2339724.318467544)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3f06810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8101fa390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81017a090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ae1860>, 2339726.229959871)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810725310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ae0670>, 2339724.146644438)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81017bfd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8101f8510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ae2d60>, 2339723.937551958)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8101f9690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810130210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ae0360>, 2339724.654141725)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810130c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810698090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ae25f0>, 2339725.024916833)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8102678d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81086f210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ca2120>, 2339725.202227494)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3e72e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3eba9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810421390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3f24190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3d1e310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81025c4b0>, 2339725.223674599)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810422650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81025f5b0>, 2339724.714539621)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3f26f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c812416270>, 2339724.987234389)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81126ed90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3ecbb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81025e4a0>, 2339724.56495425)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3eca8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3d608d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3c6e590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81025ef90>, 2339724.997997286)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3d611d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d395da90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2e00830>, 2339725.639609774)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d395e190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3c43750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3ab0c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81129bb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81025dfd0>, 2339724.796640201)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3e68150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d30bf7e0>, 2339725.56337048)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3c6f150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2e00050>, 2339725.662380659)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3c432d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2e02120>, 2339725.393289075)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d372bb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ca0a60>, 2339724.090976986)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810341950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3b27910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2e01b70>, 2339726.797268544)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3b25110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8101b1450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3c75a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2e00e50>, 2339726.501664926)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8101b2090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2e01160>, 2339726.947363626)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3c74950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3c00ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2e03ee0>, 2339727.041318441)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3c02710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3ae4e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d39a7ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81025dc50>, 2339726.740228652)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d39a6350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3b72dd0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2e02f20>, 2339726.904935411)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3df1290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3b7a210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ca1f60>, 2339726.939278777)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3b79a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d39d5f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81045ae40>, 2339727.012235214)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3bb5a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810cf6010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810459160>, 2339727.477874001)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810cf42d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d39cac10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d37cc810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810677700>, 2339727.478937325)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810159b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810677000>, 2339727.473565526)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8101bef50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3d7d150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810674ad0>, 2339727.497858441)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81262ce90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810462350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d37bc8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2e012b0>, 2339726.96752068)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3ae4710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d30bd940>, 2339727.454171067)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811a5a610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0181710>, 2339727.446510799)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d37bc9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3a204d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0183e00>, 2339728.266616838)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8111dd850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d378e3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d36642d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810676c80>, 2339727.306159017)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d378e590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d01826d0>, 2339728.109345875)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d36673d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3a02150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d01838c0>, 2339727.640833917)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3a013d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d328f810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810744ec0>, 2339728.101595829)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d328f490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d37197d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3f492b0>, 2339729.069416969)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d371b410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8101b26d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2d2e710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ae26d0>, 2339730.007304708)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3705c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3f4ac10>, 2339729.446283015)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2d2d990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d396a690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81045acf0>, 2339729.391162546)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d35a7010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3a90c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d382e4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3f4b460>, 2339729.108176007)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3a92e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3f48d70>, 2339729.740967908)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d382d850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d32ea990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d368ccd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ae0a60>, 2339729.370243756)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d32ea490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3d47af0>, 2339729.384540873)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d368f3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d35e1350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3d44f30>, 2339729.617098496)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d35e17d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811d31a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3d470e0>, 2339730.10835358)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811703cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3e70050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2e03930>, 2339730.026143462)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3a09e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3806dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d37567d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3d47070>, 2339729.580313057)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2ffaf90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3d46f20>, 2339729.880574848)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3a0a6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810f46190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0183070>, 2339729.964774942)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810f47b50>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1734c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3d45860>, 2339730.775658349)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1737710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3a226d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2fcd3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3e494e0>, 2339730.375842564)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3666c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3e4b5b0>, 2339730.429996084)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2fcc390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3ca6290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3e498d0>, 2339757.132989326)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3ca5e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2ff9890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3e4b230>, 2339730.798301479)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3223050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3631790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d35e0ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3222d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3a6ff10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0182740>, 2339729.486372229)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d396ca50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3d45b00>, 2339729.514118838)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d38bd850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2e9f450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81025c1a0>, 2339729.73507914)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2f23390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3e49da0>, 2339756.768762933)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d32eb150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3b294e0>, 2339757.169945687)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d35e3010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2ce1ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2c7fc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3b28980>, 2339757.114224776)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d382ea90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3e4baf0>, 2339756.506933708)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2c7d7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2aaeb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3b2bf50>, 2339743.919033101)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2aae850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2f6d490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2e62a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3b2a120>, 2339743.986266028)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2c7e550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2db4610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d32b6650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3b2a7b0>, 2339753.764731043)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2db4950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3b286e0>, 2339732.16558168)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d32b4f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1866f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3c2bb60>, 2339756.981131209)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1866b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3138890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d18da9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3e49860>, 2339757.095370436)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d313a750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3c2bf50>, 2339743.966090896)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d18dae10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2b2a5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3c2a580>, 2339744.093741358)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2b2ba50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2b68710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d174e250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3c2a9e0>, 2339757.126037883)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d19b4990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d18f1f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3c29860>, 2339753.799716079)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d35e35d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3b2ab30>, 2339756.732234678)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2e62d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2f46c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3e4ab30>, 2339757.081117974)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d174cb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d313a310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1828650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3b2a900>, 2339756.870333514)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d38bd4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3e4bb60>, 2339754.913799748)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d180d210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3c28ad0>, 2339754.945745941)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d18281d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d35e1510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3502dd0>, 2339754.365594243)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1866c10>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1dd86d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2c9b9a0>, 2339756.936897255)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1dd9a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1822f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1a6d610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1b25010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1ba2950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3e48830>, 2339758.287133828)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1ba12d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3c2aeb0>, 2339757.07917029)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d18217d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d38f5940>, 2339759.596789371)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1a6e010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2c988a0>, 2339757.049743917)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1b24a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1a5d890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1cac210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d38f7d90>, 2339757.116136425)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1a5ddd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3d45fd0>, 2339760.256230666)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1cad6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d310ca10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3b2ac10>, 2339760.223102279)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3706550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1c4bf50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2c99400>, 2339759.740375773)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1c4acd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2f91d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d38f6350>, 2339760.135323166)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1c07890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1c60d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d380a430>, 2339760.187647812)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1c60150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1d5a5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3f4a2e0>, 2339760.420548199)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1d58050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1d82c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3e49e10>, 2339760.490336888)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1d824d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1e234d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3500980>, 2339760.155061845)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1d8bcd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1e1bbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1ca6290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1c14110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1db2d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1d4b950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3d46cf0>, 2339760.096601856)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1e1a8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3d45be0>, 2339760.605182106)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2f96190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d380b0e0>, 2339760.120811626)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1c17b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3809780>, 2339760.156433283)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1db2950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d380a120>, 2339760.063811421)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1d22650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1cba110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1b70410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d38f4210>, 2339760.306670814)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1cba590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d35000c0>, 2339760.191483947)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1838610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1f916d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ef1550>, 2339761.403673837)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1f90110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d21b2690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110ef700>, 2339762.346198811)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d21b0050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1c72010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ef0bb0>, 2339762.509682373)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d21c3e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d208ee90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d364b110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1fbd250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d203f2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3809630>, 2339761.603102087)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d32b4e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ef1630>, 2339760.14277325)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d364a7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3d47ee0>, 2339760.182127927)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1fbdd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d380aac0>, 2339763.089907188)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d208eb90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810e9e110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3c2a040>, 2339763.317248645)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1a00c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d36488d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ef3c40>, 2339763.054744092)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3649490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1e5c210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e51010>, 2339763.852283413)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d22c1c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2130610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e51080>, 2339763.506581427)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d21330d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2393690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e50f30>, 2339763.594247095)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2392890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2d82490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8115e1550>, 2339764.390227058)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2d81150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d22464d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8115e1c50>, 2339763.890560259)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2246350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d249cad0>\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      " {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2d9fe10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1d82950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2184d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d22c3fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2161d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2225910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2154610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3809c50>, 2339763.897748216)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2d9c690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ef3ee0>, 2339763.183151442)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1f65850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ef2b30>, 2339763.470256048)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2157a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c88b173110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2276450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ef0910>, 2339763.411918155)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2184a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e53620>, 2339762.935832655)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c88b173750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0180de0>, 2339763.175407758)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2275250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e53150>, 2339763.108830002)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2160b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e50440>, 2339762.919694317)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1fe7090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d23f4ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e53a10>, 2339764.396540904)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d23ea790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e538c0>, 2339764.149794936)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d22254d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8115e1160>, 2339764.495888542)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d249d690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d21c0490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e52890>, 2339764.941969421)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2477710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1f28390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8115e0e50>, 2339765.405802146)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1fd7450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d23489d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e519b0>, 2339765.115113119)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d234b010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2474110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d237c910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3e4a740>, 2339764.866962615)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2476c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8115e01a0>, 2339765.788849763)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d237fc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d213ef50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2434290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110ee580>, 2339765.235146661)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d213c350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2428450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810557620>, 2339764.888369782)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d242ac90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2417350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d30e9950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110ecad0>, 2339765.745223722)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d24174d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810557690>, 2339765.265827428)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d30e8310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d25d91d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d24aadd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810557d20>, 2339765.67403449)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d24aa150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d24b8950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d25cab50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3500c20>, 2339765.291143979)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d25a3cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8105544b0>, 2339766.463108409)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d25bc4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d37c3190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d25414d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810554d00>, 2339766.147048657)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d37c0a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810556820>, 2339765.742716941)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2541790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e50ec0>, 2339765.209346579)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2437350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2c99860>, 2339769.477167265)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d25db610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2d830d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2570d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8115e1240>, 2339766.123188725)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2d81550>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8105551d0>, 2339766.485878523)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2573ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2506210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d26bc9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810554520>, 2339765.980026933)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d26f5850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2528cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d269e450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810143070>, 2339767.119133574)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d205c490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810140fa0>, 2339767.043813076)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d269e090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d27ea550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8101437e0>, 2339767.452482222)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d27ea710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d38c7a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8101415c0>, 2339767.906834333)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d38c66d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d26ab250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d30f3010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810142120>, 2339767.919605351)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d36cf490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810142580>, 2339767.607584423)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d311ec50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2685a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2893490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8101401a0>, 2339767.76820248)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2685790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2c9a660>, 2339767.108813868)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2891f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810ff4250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110ed080>, 2339767.197884657)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2eb26d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2eb5650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ef1fd0>, 2339767.292598228)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2eb5250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2e0b110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d37c3110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810556c10>, 2339768.877150061)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3504a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d35016a0>, 2339766.828308966)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2d4afd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d390c210>, 2339767.676808114)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d37e44d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d390e350>, 2339768.128106581)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d37c3150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d27b6790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d390ed60>, 2339768.159039032)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d27b70d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d28b2050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d271ff90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d390f770>, 2339768.372234873)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d28b0910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2e6e790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d390d390>, 2339769.475174235)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d27b7610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d28b5910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810141d30>, 2339769.722744647)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d28b4b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d28a6410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8101402f0>, 2339769.181312277)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d25fae10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2770790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1da81a0>, 2339769.384781304)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2770650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2869a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2ff2f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1da8130>, 2339769.527081858)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d286a750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1daa510>, 2339769.875375106)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2873cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d356c950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810557770>, 2339769.583685997)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d356c610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2ccc210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1daba10>, 2339769.31135755)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2ccda10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2cd8f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d27811d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d390c750>, 2339768.422569174)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d271ee50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2c50bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1daacf0>, 2339771.011813018)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2cdb390>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d390c830>, 2339767.667787181)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d27818d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1da9390>, 2339771.423186029)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2c51550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2cbea10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1da8a60>, 2339770.010551755)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d28001d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2830610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8115e2190>, 2339771.117990739)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d28317d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d38c65d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ea52b0>, 2339771.151623989)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d38c4910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d29d9850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ea7850>, 2339770.659413846)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d29d86d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2bb9fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ea75b0>, 2339771.77448425)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d21b0d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2989990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ea7930>, 2339770.409269042)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2989850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2bb8290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110ef5b0>, 2339770.915786897)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2bb8b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2929110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ea6900>, 2339771.637592673)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d292b650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2da5410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1fc6890>, 2339771.737422328)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2eb7f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2d13650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2f16f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810141390>, 2339771.766831845)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d28f2d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1fc66d0>, 2339771.119346994)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2f15d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2933310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2c43b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1da96a0>, 2339771.477688393)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2932350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1fc6cf0>, 2339771.555391428)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2c41150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2c15c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2e336d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d327c950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ea58d0>, 2339771.162926559)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d327e910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8105543d0>, 2339772.838621831)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2d426d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3b28a60>, 2339769.560834785)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2e32b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2729f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ea7cb0>, 2339772.923299863)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d272b090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2c1e850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1fc7af0>, 2339773.647894348)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2c1f7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d29a70d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1fc6200>, 2339773.310245936)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2b817d0>\n",
      "Evaluating workflow:  50%|████▉     | 1485/3000 [05:03<09:46,  2.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2026-01-13 12:47:37.178\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1487/3000 [05:03<07:28,  3.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|████▉     | 1491/3000 [05:04<05:16,  4.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|████▉     | 1495/3000 [05:04<03:25,  7.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1497/3000 [05:04<02:53,  8.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 1502/3000 [05:05<03:49,  6.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 1506/3000 [05:05<02:54,  8.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1509/3000 [05:06<04:06,  6.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1511/3000 [05:06<04:26,  5.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1513/3000 [05:07<03:43,  6.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1516/3000 [05:07<03:42,  6.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1519/3000 [05:08<04:02,  6.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1522/3000 [05:08<03:05,  7.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1527/3000 [05:08<02:27,  9.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1529/3000 [05:08<02:26, 10.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1531/3000 [05:09<02:18, 10.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1534/3000 [05:09<02:44,  8.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1536/3000 [05:09<03:26,  7.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████▏    | 1541/3000 [05:10<02:36,  9.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████▏    | 1544/3000 [05:10<02:08, 11.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1546/3000 [05:10<02:28,  9.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1550/3000 [05:11<02:30,  9.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1552/3000 [05:11<02:23, 10.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1554/3000 [05:11<03:07,  7.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1559/3000 [05:12<02:39,  9.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1565/3000 [05:13<02:30,  9.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1567/3000 [05:13<03:13,  7.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1571/3000 [05:13<02:35,  9.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1573/3000 [05:14<02:20, 10.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1579/3000 [05:14<02:40,  8.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1581/3000 [05:15<03:02,  7.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1583/3000 [05:15<03:00,  7.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1585/3000 [05:16<04:39,  5.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1587/3000 [05:16<04:32,  5.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1593/3000 [05:16<02:00, 11.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1597/3000 [05:17<01:27, 15.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1600/3000 [05:17<01:34, 14.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1602/3000 [05:17<01:43, 13.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1604/3000 [05:18<04:04,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▎    | 1606/3000 [05:18<03:56,  5.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▎    | 1611/3000 [05:19<02:44,  8.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1615/3000 [05:19<02:05, 11.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1617/3000 [05:20<03:42,  6.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1624/3000 [05:20<02:19,  9.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1627/3000 [05:21<02:38,  8.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1631/3000 [05:21<02:27,  9.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1633/3000 [05:22<03:47,  6.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▍    | 1637/3000 [05:22<02:50,  8.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1640/3000 [05:22<02:15, 10.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1642/3000 [05:22<02:32,  8.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1646/3000 [05:23<02:27,  9.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1648/3000 [05:23<02:46,  8.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1651/3000 [05:23<02:21,  9.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1653/3000 [05:24<02:38,  8.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▌    | 1656/3000 [05:24<03:08,  7.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▌    | 1663/3000 [05:25<01:41, 13.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1665/3000 [05:25<02:26,  9.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1670/3000 [05:26<02:30,  8.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1672/3000 [05:26<03:23,  6.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1678/3000 [05:27<02:32,  8.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1680/3000 [05:27<03:30,  6.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1681/3000 [05:27<03:34,  6.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1686/3000 [05:28<02:25,  9.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▋    | 1689/3000 [05:28<01:49, 11.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▋    | 1695/3000 [05:29<02:18,  9.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1697/3000 [05:29<02:06, 10.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1702/3000 [05:30<02:21,  9.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1704/3000 [05:30<03:30,  6.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1708/3000 [05:31<02:34,  8.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1712/3000 [05:31<02:18,  9.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1715/3000 [05:31<01:49, 11.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1717/3000 [05:32<02:33,  8.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1720/3000 [05:32<02:40,  7.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1724/3000 [05:33<03:07,  6.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1726/3000 [05:33<02:35,  8.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1731/3000 [05:33<02:00, 10.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1734/3000 [05:34<03:09,  6.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1736/3000 [05:34<03:11,  6.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1741/3000 [05:35<02:14,  9.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1743/3000 [05:35<02:21,  8.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1745/3000 [05:36<03:52,  5.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1746/3000 [05:38<09:33,  2.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:48:12.154\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 10 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1747/3000 [05:38<08:48,  2.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:48:12.162\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 10 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:48:12.169\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to GBF1 and then measure expression of CD58. Does this perturbation cause a significant change in CD58 expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to GBF1 and then measure expression of CD58. Does this perturbation cause a significant change in CD58 expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 2494}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 10 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2026-01-13 12:48:12.359\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in gene regulation studies. For experiments performed in K562 cells, DDOST is perturbed and C9orf64 expression is measured. Determine whether C9orf64 exhibits a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are an expert in gene regulation studies. For experiments performed in K562 cells, DDOST is perturbed and C9orf64 expression is measured. Determine whether C9orf64 exhibits a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 3879}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 10 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1751/3000 [05:38<04:45,  4.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:48:12.370\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in single-cell biology and functional genomics. In K562 cells, MRPL39 is perturbed and the expression of HIST1H2AM is measured. Does this perturbation cause a significant change in HIST1H2AM expression? Answer strictly in the format 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are an expert in single-cell biology and functional genomics. In K562 cells, MRPL39 is perturbed and the expression of HIST1H2AM is measured. Does this perturbation cause a significant change in HIST1H2AM expression? Answer strictly in the format 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 2420}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 10 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:48:12.413\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 10 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:48:12.431\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 10 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:48:12.442\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a genomics expert evaluating perturbation experiments. In K562 cells, UFL1 is perturbed and the expression of KDM1B is measured. Determine whether KDM1B shows a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are a genomics expert evaluating perturbation experiments. In K562 cells, UFL1 is perturbed and the expression of KDM1B is measured. Determine whether KDM1B shows a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 1166}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 10 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:48:12.534\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are assisting with the interpretation of perturbation-based expression data. In K562 cells, DDIT3 is perturbed and STC2 expression is quantified. Does this perturbation result in a significant change in STC2 expression compared with control cells? Respond exactly as 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are assisting with the interpretation of perturbation-based expression data. In K562 cells, DDIT3 is perturbed and STC2 expression is quantified. Does this perturbation result in a significant change in STC2 expression compared with control cells? Respond exactly as 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 182}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 10 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▊    | 1756/3000 [05:38<02:48,  7.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:48:12.545\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in single-cell biology and functional genomics. In K562 cells, EIF2B4 is perturbed and the expression of GDF11 is measured. Does this perturbation cause a significant change in GDF11 expression? Answer strictly in the format 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are an expert in single-cell biology and functional genomics. In K562 cells, EIF2B4 is perturbed and the expression of GDF11 is measured. Does this perturbation cause a significant change in GDF11 expression? Answer strictly in the format 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 390}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 10 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▊    | 1758/3000 [05:41<07:39,  2.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:48:21.778\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert analyst of perturbation datasets. For K562 cells subjected to perturbation of KCTD16, does the expression profile of BACH2 indicate a significant change relative to control conditions? Reply strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are an expert analyst of perturbation datasets. For K562 cells subjected to perturbation of KCTD16, does the expression profile of BACH2 indicate a significant change relative to control conditions? Reply strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 3477}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 13 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:48:22.228\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 12 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▊    | 1760/3000 [05:48<23:29,  1.14s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:48:22.286\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in regulatory genomics. Consider data from K562 cells in which CARS is perturbed and NAV2 expression is observed. Does this perturbation lead to a significant difference in NAV2 expression relative to control conditions? Answer strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are an expert in regulatory genomics. Consider data from K562 cells in which CARS is perturbed and NAV2 expression is observed. Does this perturbation lead to a significant difference in NAV2 expression relative to control conditions? Answer strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 4863}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 12 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:48:22.446\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in gene regulation studies. For experiments performed in K562 cells, GMPPB is perturbed and OR2T6 expression is measured. Determine whether OR2T6 exhibits a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are an expert in gene regulation studies. For experiments performed in K562 cells, GMPPB is perturbed and OR2T6 expression is measured. Determine whether OR2T6 exhibits a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 1849}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 12 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▊    | 1762/3000 [05:48<18:06,  1.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:48:22.593\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are assisting with the interpretation of perturbation-based expression data. In K562 cells, CAD is perturbed and AC008074.3 expression is quantified. Does this perturbation result in a significant change in AC008074.3 expression compared with control cells? Respond exactly as 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are assisting with the interpretation of perturbation-based expression data. In K562 cells, CAD is perturbed and AC008074.3 expression is quantified. Does this perturbation result in a significant change in AC008074.3 expression compared with control cells? Respond exactly as 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 1902}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 12 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:48:22.593\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a genomics expert evaluating perturbation experiments. In K562 cells, MTHFD1 is perturbed and the expression of KCNH7 is measured. Determine whether KCNH7 shows a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are a genomics expert evaluating perturbation experiments. In K562 cells, MTHFD1 is perturbed and the expression of KCNH7 is measured. Determine whether KCNH7 shows a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 1786}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 12 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1763/3000 [05:48<15:48,  1.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:48:22.811\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a genomics expert evaluating perturbation experiments. In K562 cells, DERL2 is perturbed and the expression of PDK2 is measured. Determine whether PDK2 shows a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are a genomics expert evaluating perturbation experiments. In K562 cells, DERL2 is perturbed and the expression of PDK2 is measured. Determine whether PDK2 shows a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 2166}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 12 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1766/3000 [05:50<16:25,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:48:34.508\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:48:34.512\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1767/3000 [06:00<54:04,  2.63s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:48:34.566\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:48:34.572\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:48:34.620\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1771/3000 [06:00<25:15,  1.23s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:48:34.693\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:48:34.924\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a functional genomics specialist. In K562 cells, we perturb FECH and monitor HERPUD1 expression. Decide whether this perturbation leads to a significant alteration in HERPUD1 expression. Answer only in the format 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are a functional genomics specialist. In K562 cells, we perturb FECH and monitor HERPUD1 expression. Decide whether this perturbation leads to a significant alteration in HERPUD1 expression. Answer only in the format 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 4438}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1773/3000 [06:01<19:05,  1.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:48:35.124\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:48:35.153\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1775/3000 [06:01<14:18,  1.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1777/3000 [06:02<14:46,  1.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1778/3000 [06:03<13:26,  1.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:48:37.290\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1779/3000 [06:03<11:42,  1.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1781/3000 [06:03<08:30,  2.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1784/3000 [06:04<05:07,  3.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1785/3000 [06:04<05:07,  3.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|█████▉    | 1791/3000 [06:04<02:50,  7.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|█████▉    | 1794/3000 [06:05<03:12,  6.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1795/3000 [06:05<04:17,  4.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|██████    | 1800/3000 [06:06<02:56,  6.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1802/3000 [06:07<04:01,  4.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|██████    | 1804/3000 [06:07<04:06,  4.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|██████    | 1809/3000 [06:07<02:12,  8.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1811/3000 [06:08<01:56, 10.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1814/3000 [06:08<01:40, 11.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1816/3000 [06:08<02:00,  9.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1821/3000 [06:09<03:00,  6.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1823/3000 [06:09<02:39,  7.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1825/3000 [06:10<02:17,  8.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1829/3000 [06:10<02:24,  8.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1831/3000 [06:10<02:13,  8.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1835/3000 [06:11<02:09,  9.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████▏   | 1840/3000 [06:11<01:57,  9.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████▏   | 1844/3000 [06:12<01:42, 11.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1846/3000 [06:12<02:10,  8.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1850/3000 [06:12<02:03,  9.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1853/3000 [06:13<02:30,  7.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1859/3000 [06:13<01:41, 11.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1861/3000 [06:13<01:35, 11.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1863/3000 [06:14<02:32,  7.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1867/3000 [06:14<02:20,  8.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1869/3000 [06:15<02:25,  7.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1870/3000 [06:15<02:34,  7.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▎   | 1875/3000 [06:15<02:12,  8.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1878/3000 [06:16<02:22,  7.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1880/3000 [06:16<02:07,  8.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1882/3000 [06:17<03:12,  5.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1886/3000 [06:17<02:21,  7.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1892/3000 [06:17<01:41, 10.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1897/3000 [06:18<01:25, 12.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1902/3000 [06:19<02:20,  7.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▎   | 1905/3000 [06:19<01:47, 10.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▎   | 1909/3000 [06:19<01:58,  9.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1915/3000 [06:20<01:58,  9.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1917/3000 [06:21<02:26,  7.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1921/3000 [06:21<02:01,  8.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1923/3000 [06:21<01:51,  9.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1925/3000 [06:22<02:23,  7.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1927/3000 [06:22<02:49,  6.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1931/3000 [06:22<02:07,  8.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1933/3000 [06:23<01:52,  9.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▍   | 1938/3000 [06:23<01:38, 10.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1940/3000 [06:23<01:29, 11.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1942/3000 [06:24<02:07,  8.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1944/3000 [06:24<02:17,  7.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1947/3000 [06:24<01:56,  9.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1949/3000 [06:24<02:02,  8.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 1955/3000 [06:25<01:28, 11.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1957/3000 [06:25<01:28, 11.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1959/3000 [06:25<01:54,  9.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1961/3000 [06:26<02:35,  6.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1965/3000 [06:26<02:00,  8.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1968/3000 [06:26<01:31, 11.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1970/3000 [06:27<02:37,  6.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1972/3000 [06:27<02:53,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1974/3000 [06:28<02:28,  6.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1976/3000 [06:28<02:41,  6.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1980/3000 [06:28<02:05,  8.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1982/3000 [06:29<02:02,  8.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1985/3000 [06:29<01:33, 10.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▋   | 1989/3000 [06:29<02:07,  7.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▋   | 1994/3000 [06:30<01:47,  9.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 1997/3000 [06:30<01:38, 10.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 2001/3000 [06:31<01:58,  8.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 2007/3000 [06:31<01:41,  9.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2009/3000 [06:32<01:33, 10.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2011/3000 [06:32<01:52,  8.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2013/3000 [06:33<03:19,  4.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2014/3000 [06:33<03:22,  4.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d297e6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ea4c20>, 2339773.857815459)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3140d10>\n",
      "Evaluating workflow:  67%|██████▋   | 2016/3000 [06:34<04:00,  4.08it/s]Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3143f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d20bc4b0>, 2339773.441707442)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3142b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2c425d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1fc7460>, 2339774.501761417)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2c414d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7d0f450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810140590>, 2339773.769332357)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7d0e550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811ca35d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d390c980>, 2339774.458902497)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d297ed50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c76436d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7ee91d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d272ba90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c76426d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1fc4210>, 2339773.865041418)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d272b910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ea73f0>, 2339777.013900908)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c77cad90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d21bec10>, 2339773.113772514)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7ee9810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d20be740>, 2339776.869631546)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7643350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d29755d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d22c65f0>, 2339775.917957109)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c762a090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c76e5a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d23d6b30>, 2339777.839381105)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c76e7810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c761a4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7685ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c76a6090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7557710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1fc46e0>, 2339775.735063419)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2e32d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d20bd010>, 2339777.113227699)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c760de90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d22c5d30>, 2339776.737330334)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c76a5e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d23d4130>, 2339777.876936621)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7557510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c759f810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3263ee0>, 2339778.476096488)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c759f6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c75c5c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d21bda20>, 2339777.952475487)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c75c5ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c74a2dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d24d6d60>, 2339781.186924448)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c74a2350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c73c3b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d288f460>, 2339781.837587478)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c73c2590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7392ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d25d01a0>, 2339782.523570219)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c763fbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7232810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d25d0980>, 2339782.661673497)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c73f4210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c720c290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d24d5e80>, 2339783.10890223)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c76fdd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c74a33d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d27ef310>, 2339782.163686465)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c74a3d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c72ef850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d27ef230>, 2339783.667386535)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c72efd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c714a6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7152c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d27eca60>, 2339785.522886444)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c714bcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d27ee3c0>, 2339784.482361435)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7152b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c72eff50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2747c40>, 2339784.463416406)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c71eee10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c70449d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d29d1710>, 2339784.793204991)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7045010>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7258d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d25d1160>, 2339783.349045301)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7259310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c71d3f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d25d3a80>, 2339783.67034336)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c71d3e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c70a86d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d29d2270>, 2339785.588722659)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c70a8950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7180f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d29d2510>, 2339790.066967925)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c70bbcd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c70d6a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2b644b0>, 2339786.670987903)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6f2a790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6fbb190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6f2a6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2b64a60>, 2339786.858220543)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6fbb450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2747150>, 2339787.786020465)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6f2bc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7180590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d29d2120>, 2339787.443246319)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c71832d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c70f4750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2a364a0>, 2339789.095806292)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c70f4690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6fe2050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6e56650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2a35780>, 2339787.461275186)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6fe2550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2a37850>, 2339787.408225154)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6e56b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6d50790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2b675b0>, 2339790.71303055)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6d50a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6d09650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2f32510>, 2339790.337971465)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6d08310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6f6c190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2dbd6a0>, 2339790.550775902)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6da0a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6d97f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2dbe4a0>, 2339790.708426742)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6d975d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6d12d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2a355c0>, 2339789.176938389)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6f6d850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6c3b210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d198f1c0>, 2339817.317440568)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6c3ba90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6c87090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d198c520>, 2339791.914684521)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6c86fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6caff10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d198dd30>, 2339804.955407919)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6c17e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6c79e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2a366d0>, 2339816.555011099)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6c7a0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6cf6810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d198d940>, 2339816.535760562)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6b1ce10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6c9fd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d198fb60>, 2339791.214036716)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6c9fc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6abddd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2dbe820>, 2339804.737484754)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6abdc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6bbd9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6d61b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6bda6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6beba90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6bfe6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6a07850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6c1c5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1b97000>, 2339817.40727337)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6bbd890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1b95400>, 2339804.278022442)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6d60a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d25d0520>, 2339793.600017016)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6bdbdd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1b967b0>, 2339804.464358987)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6bea090>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1b944b0>, 2339804.347613602)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6bfee10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1b94c90>, 2339804.356966559)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6a07650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c7e80b40>, 2339794.517566004)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6a1d190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6ada7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2a35a20>, 2339819.202305666)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6ada290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6b4ec50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d198e270>, 2339819.810539305)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6a05390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6999210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c697d390>, 2339820.415209088)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6969250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c69c1f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c697dd30>, 2339820.323999668)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c69c1c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c69ecb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c697e270>, 2339817.956547857)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6bfce50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c68544d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6a66910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6a1ce50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c694e950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c69b97d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c6ae2200>, 2339820.863878511)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6a1ead0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c6ae3690>, 2339820.568004752)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6854d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1b955c0>, 2339817.418430828)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6efd250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c6ae3770>, 2339817.094615602)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c694ee10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c697d940>, 2339820.664832025)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c69b9650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c696abd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c68044d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c7e814e0>, 2339817.687744161)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c696b510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1b96190>, 2339818.082226201)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c68059d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c687c910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c6ae39a0>, 2339819.681822588)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c687cdd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c68a4050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c7e83e00>, 2339820.558064299)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c68a4550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c68d5e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c683b9a0>, 2339820.811202034)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c68d5cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6731390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c6715390>, 2339820.796866515)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6731190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6738b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2b64910>, 2339819.682916116)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6746a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c67781d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c6716190>, 2339822.178734565)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c677b390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6798a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c6717000>, 2339820.738334184)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6798890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c67c0ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c68b2e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c68ee410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c670dc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c683aa50>, 2339820.257598177)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c68b2c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c697f7e0>, 2339820.669760133)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c68ee290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c6714830>, 2339820.335495029)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c670e4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2f419d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2abcdd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2a5b6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2d24210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2a84590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2f9db10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2a1afd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811c39d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d07f1190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2ba8dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c778db50>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c77b4c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c77e1c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c77fdf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c76288d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c77b5f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c77c9450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c77acbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7662490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7680cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c76ca910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c76db450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c76f9950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c77fdf10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c757a6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c76c0bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c753fc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c759d110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c75e2090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c75b7150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c75ffcd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c744a9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c744b6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c747ab50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c75004d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c76fdb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c74a3c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c75f7c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7419fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c745de10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c74c94d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c74cb590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c74fc890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7304bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c74797d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c741b710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7362d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7388a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c734e390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7301e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c73cbc50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c73ee450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7501950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c74249d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c721c590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c726b190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7d0b8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c72a2910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c72b2910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c721db90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c711f790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7278850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c710fc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c73c24d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c71a8d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c71b7550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c71809d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c71ecdd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c700e490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7046250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c72d8b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c703b1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c70696d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7089310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c709e590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c71b7f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c70e7490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6f25d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6f637d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c71c7e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6fb8690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c70fd8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6fa7e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6fd8bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c70fe750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6e20a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6e3eb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6e97e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7039b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6ec7450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c70d6d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6ef2210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6e60450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6e838d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6d26d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6d60950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6e4c3d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6dbcbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6d2d290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6de7590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6c06690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6ea7550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6d43010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6de1950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6c54650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6c5ced0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7181310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6b2aad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6b42750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6b31ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6b74610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6b7e410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6b95510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6cd6310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6f0ff90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c690ced0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c682aed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c695d310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c6839f60>, 2339822.224849248)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6783190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1fc74d0>, 2339772.682769339)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2f43910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d20bc8a0>, 2339772.900830032)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2abfc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d20bdef0>, 2339772.502851468)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2a5ba90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d20bc910>, 2339773.093045088)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2d242d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d20bda20>, 2339773.026974062)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2fe7410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d20be970>, 2339773.452686071)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2c65ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d20bd6a0>, 2339773.600208591)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2a1a350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ea62e0>, 2339773.682952969)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811c3ac10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d21bfd90>, 2339774.42565381)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d07f13d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d20be120>, 2339774.507668635)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2baae90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1fc7930>, 2339775.046225848)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c778da10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d21bcd70>, 2339775.299475696)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c77b7410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ea70e0>, 2339776.168747487)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c77e22d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d20bc750>, 2339776.220499952)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c77fdd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d22c6660>, 2339776.29699202)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d25f8910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d390d7f0>, 2339774.915501917)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c77b5d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d21bd5c0>, 2339775.329435189)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c77c9090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d22c7e70>, 2339774.844015197)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c77ad5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d22c7cb0>, 2339775.483834036)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7662290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d22c7770>, 2339776.164238364)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2b49290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d22c4520>, 2339777.342255656)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c76cafd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d23d58d0>, 2339776.639609467)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c76db310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d23d47c0>, 2339777.518801612)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c76f9810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d23d6740>, 2339777.497730595)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2f06bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d23d5d30>, 2339777.029190681)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c757a650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d23d5010>, 2339777.699151808)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c756df50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d23d56a0>, 2339776.733461373)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c753fb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d21bdbe0>, 2339777.77139979)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c759d850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3262200>, 2339778.225479423)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2a86bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3261630>, 2339778.55068301)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c75b7690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d21bc600>, 2339779.028916764)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c75fdfd0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d24d5c50>, 2339778.747426359)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c744b1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d24d7000>, 2339779.17861446)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7596910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d24d5da0>, 2339779.59707545)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c747a950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1fc6660>, 2339778.820754879)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c756f2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3263690>, 2339779.329963187)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c76fd250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d24d6190>, 2339779.529085901)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c74a3b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3261f60>, 2339778.402032682)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c75f7b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1dab850>, 2339778.468047356)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7419e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d24d4050>, 2339778.912870741)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c745dcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d24d5710>, 2339779.753124998)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c754a410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1fc6190>, 2339779.368337214)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c74cb390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d24d6430>, 2339780.76418522)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c74fcf50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d24d6c80>, 2339779.799267976)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7304790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d23d73f0>, 2339780.162466642)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c75e2450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d288fa10>, 2339780.231537636)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c741b850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d23d4a60>, 2339780.775959486)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c74546d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d288c210>, 2339780.830199973)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c73890d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d288e820>, 2339781.922420312)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c734e290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d288fc40>, 2339781.252250854)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c739ba10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d288e890>, 2339781.030413756)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c73cbad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d25d0d70>, 2339782.228652105)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c73eeb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d25d0670>, 2339781.057811101)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c754a910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d21bd4e0>, 2339779.757721183)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c74d9b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d25d3620>, 2339780.863708448)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c721c390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d25d17f0>, 2339781.857251921)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c726af90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d25d2120>, 2339782.238357475)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7333810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d25d1320>, 2339783.080906696)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c72a2fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d27ecfa0>, 2339782.662871387)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c72b2710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d27ec050>, 2339783.702609873)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c739bc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d288e2e0>, 2339783.299527855)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c711f590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d25d27b0>, 2339781.825511042)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7278790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d27ef8c0>, 2339782.667597802)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c710fad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2746430>, 2339783.668463868)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7333190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2745400>, 2339784.131854325)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c71a8c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2744d70>, 2339784.104667476)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c71b7350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2745da0>, 2339784.911765426)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7180d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2744de0>, 2339785.507823038)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c71ed510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d27ef930>, 2339784.358726223)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c700fc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d288e510>, 2339785.069520644)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7046810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d29d1940>, 2339787.18858789)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c702dc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d29d3690>, 2339784.786906382)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c726b5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d29d1780>, 2339784.822959249)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c707cf90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d29d2430>, 2339785.570063884)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7089110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d29d0980>, 2339785.624619378)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c709e390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d29d3380>, 2339785.6309163)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c71b7e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d29d33f0>, 2339787.338607329)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c70e7350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2b66740>, 2339786.219447133)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6f26bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2b64440>, 2339787.429293521)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6f635d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2b66040>, 2339787.958664388)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c71c7b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2b67700>, 2339786.945057907)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6fb9610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d29d2200>, 2339785.760740286)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c70fd790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d25d16a0>, 2339786.692170636)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6fa7c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2b65cc0>, 2339786.697686545)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6fd9050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2746e40>, 2339787.547879467)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c70fe8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d27469e0>, 2339787.946715525)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7253690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2a35240>, 2339787.849590377)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6e3f2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2a37700>, 2339788.320129464)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6e97d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2f32ac0>, 2339789.067330982)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c703a650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2f31470>, 2339789.157082895)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6ec7250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2b64980>, 2339788.956040311)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6ee7f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2f33770>, 2339788.994194965)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6ef3f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2a36e40>, 2339788.144790007)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6e60410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2a34670>, 2339788.226957858)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6e836d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2b66430>, 2339789.656226683)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c715ed90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2f33b60>, 2339789.637231085)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6d60e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d29d3ee0>, 2339789.79200546)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7253210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2dbd4e0>, 2339790.662242502)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6dbd050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2f30d00>, 2339790.146664822)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6d2dc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2dbe900>, 2339791.180594455)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6de7390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d27446e0>, 2339791.126940515)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6c06550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2746510>, 2339791.150317924)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6ea4290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2747e00>, 2339789.509056588)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6d436d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2b64130>, 2339790.274015813)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6de2050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2dbf230>, 2339791.197253387)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6c54d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2a340c0>, 2339791.224613493)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6c5cd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d198d630>, 2339804.766704413)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7181210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d198c130>, 2339804.813190648)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6b2ae50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d188d400>, 2339804.534931036)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6b42e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d188da90>, 2339817.249344498)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6b332d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d198c910>, 2339804.776797837)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6b74410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d198e2e0>, 2339816.457740362)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6b7eb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d188d320>, 2339816.742118376)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6b953d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d198f230>, 2339804.191671265)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6cd6910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1b94e50>, 2339817.332609171)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c745c590>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c6ae14e0>, 2339817.363130944)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6903190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c68386e0>, 2339818.703009473)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c682abd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c6ae35b0>, 2339819.1971645)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c695ce90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c72ffc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d24d6ac0>, 2339783.031740487)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c72fdf10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c668e790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c6717620>, 2339822.044625804)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c66691d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c679ab90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1b963c0>, 2339823.495651797)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c661d510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6732cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c696c7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1b94f30>, 2339823.181771476)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c67319d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1b94520>, 2339822.114022135)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c696c4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6ab9590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d198cfa0>, 2339823.323194741)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6aba3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c68ef250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6822c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c6ae11d0>, 2339823.471421452)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6744fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c6ae01a0>, 2339823.407658035)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6823350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c68bb910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c6ae04b0>, 2339822.885687706)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c68bab90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6ae6ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2dbf620>, 2339824.712293713)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6ae5250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6606050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c663b950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c66697d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6681250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6754750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c687f610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c68a9110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6aec0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c69f7550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c67c8830>, 2339821.697401198)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c66067d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c697cec0>, 2339821.816581063)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6669b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c68394e0>, 2339821.822938809)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6668a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c6838440>, 2339822.151261924)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c66329d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c6ae07c0>, 2339823.537524881)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6755850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c6ae1160>, 2339823.930045516)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c68d6290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c6ae0360>, 2339824.55186257)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c68a9710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c6ae2f90>, 2339824.548367555)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6aeec50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c67179a0>, 2339821.883003029)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c69f7390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6a62cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c660c450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c683a4a0>, 2339823.917114915)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6a61250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d27edd30>, 2339825.010185861)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c660e650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6a87210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d288c360>, 2339824.718080218)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6a846d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c668d390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d24d64a0>, 2339825.262944178)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c695c750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6a06810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6935790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8115e3380>, 2339825.049590087)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c671f090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c6716580>, 2339824.387430449)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6936950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c69c4410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c697d6a0>, 2339825.341618636)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c69c4690>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c69c4ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8115e38c0>, 2339824.507405577)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c66044d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6cc4c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e52660>, 2339825.338038915)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6bece10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6d19b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e51780>, 2339825.575396014)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6d2e110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6ee44d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c705eb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c6839240>, 2339825.928793208)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6ff7c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c6ae22e0>, 2339826.46060878)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6f93950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6ada090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6938c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6890950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8115e02f0>, 2339824.981481907)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6939010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8115e0f30>, 2339825.141129801)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6750310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c6ae2cf0>, 2339825.503301116)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6adadd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6f2bbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6aba350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8115e3460>, 2339825.499540061)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6f60710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c683b230>, 2339826.808277301)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6aba890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c668d190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c67160b0>, 2339825.99995071)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6c5d3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c701a010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e509f0>, 2339826.6675173)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6fb81d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c670f810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7152510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c683a7b0>, 2339826.371455689)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c670e590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e512b0>, 2339826.738142976)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7152a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c71d9990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e50ad0>, 2339826.80342841)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c72db3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c725ef50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7420ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e51010>, 2339826.824219172)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c696ced0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110eeba0>, 2339826.75612141)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c76ff690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c76a3a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c662b2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110ed7f0>, 2339827.253074962)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6667e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7560950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110ec670>, 2339827.344152879)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c693b8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c77e3350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c77afe50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110ec0c0>, 2339828.838801862)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c77e1a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110eec80>, 2339827.700650887)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c77ad850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2c41110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110ecd00>, 2339827.65269457)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2c40a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7527d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2976450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d288c3d0>, 2339826.962027951)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c76a0ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110ed9b0>, 2339827.176032687)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7379e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110ece50>, 2339827.736751281)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2976490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2b6df10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2933f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2c99ef0>, 2339827.872210085)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2931610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2da7110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d23d6d60>, 2339828.285679915)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2da7250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2879110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2c9b150>, 2339829.315726148)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d287bf10>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2a5c810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2c9b380>, 2339828.750100406)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7405f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d327f410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2b17a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2c9b000>, 2339828.361304144)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d327ef10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2c9b310>, 2339828.820938148)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2b15850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2e6c4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d28d1050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ef26d0>, 2339828.792461111)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2e6d210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ef3d20>, 2339829.443836502)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d28d3490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d286b210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ef0280>, 2339828.685120832)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d286b050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2700450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110edb00>, 2339830.382433223)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2703b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2e41210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d25be1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8115e1080>, 2339831.622777349)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2e414d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ef19b0>, 2339830.2445471)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d25be490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d30f0d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110ed630>, 2339828.339479995)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2b6c9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811b58a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2f1cf90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2c9b070>, 2339829.243595393)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2d11610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e514e0>, 2339827.548364858)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c766a6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c68384b0>, 2339827.968728314)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c660e6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d35400d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d267d710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ef3c40>, 2339830.177399269)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d27c6950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d27ef4d0>, 2339829.897557301)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d267e090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d27e7790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ef3e00>, 2339830.278651021)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d27e5d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2687e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d26d7010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c6716740>, 2339829.729025883)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2685750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2e03620>, 2339830.216979631)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7595850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2593410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2e00d70>, 2339830.440163495)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2592610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d286b0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2c987c0>, 2339830.43607706)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2fda6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6b408d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2e02b30>, 2339830.422748308)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2609110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c74ad010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2e00f30>, 2339831.802585001)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2e6c290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d35ef650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d30e8fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2e02190>, 2339831.156203886)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6c9d2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2e007c0>, 2339831.16803323)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d30ea850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2581f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d21b2ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3299290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2e02a50>, 2339831.26747706)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2e6d450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2330c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d30bf7e0>, 2339831.190763216)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d21b0090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110ecfa0>, 2339829.245996814)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3299590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d30bf4d0>, 2339832.92186052)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2332e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2c67e10>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d30bfd90>, 2339831.903832698)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2c67050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3284990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2c9a200>, 2339832.640174234)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d32859d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d213fed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d35e4d70>, 2339831.872796132)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d213fe10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d21d5a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d30bfb60>, 2339833.181453638)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d20844d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d20c0ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110eee40>, 2339833.575241325)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d29fbe50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d206bed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3600e50>, 2339833.363657851)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d287bf50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2a63290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3601390>, 2339833.15846122)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1e5bc50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1d4a6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d36037e0>, 2339833.741800129)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1d49ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1edf1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3505050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d237e510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1f3d890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7594ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ef2970>, 2339833.551111248)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1edc850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2c9a740>, 2339831.273475368)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1fba2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c697e200>, 2339833.176606031)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d237f090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e52740>, 2339833.223288746)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1f3dcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3600bb0>, 2339833.398432411)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1f01110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2029410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d23d4d00>, 2339833.352944753)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2028d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1d20150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c88bbe4590>, 2339833.282070681)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1d22450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d24fa590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e134d0>, 2339833.038061339)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2d04590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d19bd050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c7e82e40>, 2339833.829248282)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d19bd210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d21de0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1b94a60>, 2339835.184184184)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d21dd490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1b1e590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1e88810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c7e80e50>, 2339833.830377931)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1b1d750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c7e83ee0>, 2339833.63418341)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1e88990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d23b0b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c7e83bd0>, 2339834.807457484)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d23b1f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1ecf310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d27472a0>, 2339836.981851316)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1ec44d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1e4fa50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3600c20>, 2339835.12951069)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1e4dcd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d373b250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2e00130>, 2339835.361562947)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d25bc4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d26d6510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1bbf8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1a0fdd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1bc4450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c88bbe44b0>, 2339833.248580567)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d36bc990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e13850>, 2339833.343638117)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2d06350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0182dd0>, 2339835.114956284)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d26d4810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e532a0>, 2339835.487187527)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1bbc6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1a5ff50>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c88bbe4980>, 2339836.803349012)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1a5ddd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1e200d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d01832a0>, 2339836.819004921)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1db2690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1da50d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0181be0>, 2339835.597810128)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1d6d850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1c509d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0180ec0>, 2339835.648675678)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1c53890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d36fd190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1c14dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0182eb0>, 2339836.877731264)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1edc5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3602510>, 2339836.959673411)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1c17a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d19d1050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3901310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0180e50>, 2339836.557192498)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3738b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0180440>, 2339835.753145977)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d39023d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1cae250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1b0fb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0270050>, 2339835.446520587)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1cac950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0270130>, 2339835.716841111)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1b0cc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810e9de10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d19d88d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0270d70>, 2339836.872413642)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810e9e350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d02700c0>, 2339837.433377421)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d19d8310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2f47e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1820850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1745b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1ab9190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0183d20>, 2339836.39270103)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1745c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0272ba0>, 2339836.835212035)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2f45750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0183930>, 2339834.933685782)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1823750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0183e00>, 2339835.47787938)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1aca6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1928b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0273230>, 2339838.425397584)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d192b410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1c538d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d188e820>, 2339837.770477337)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1c53290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d35bd450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d188f620>, 2339837.403595303)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3903210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d35942d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d188ef20>, 2339837.806797727)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3595590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d38bcad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d188ee40>, 2339837.647161077)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d38bcf10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2f4f150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0181ef0>, 2339838.673084031)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2f4dc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2f21610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d188d390>, 2339838.480404474)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2f23e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3124590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110f89f0>, 2339838.545331345)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3124a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d37ac150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110fba10>, 2339838.802586675)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1d0ec50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2ffb1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3ce8650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d32c0950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d188e740>, 2339838.975775792)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d32c2350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2f96bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8106e87d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d02733f0>, 2339836.983337915)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2f96110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2c99240>, 2339839.270840677)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8106eb550>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2e02970>, 2339838.748563127)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2ff9d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d188fb60>, 2339840.088924912)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3ceb750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3224cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d368df90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d32c1890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d188e2e0>, 2339838.837226597)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3226610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d29d1240>, 2339839.690789977)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d368da10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110f9320>, 2339839.366862486)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d32c2b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d36b5250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110fa350>, 2339839.282619916)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d36b5f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1978f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110f8a60>, 2339839.236730492)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2fc8c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d176c890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3643a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110f8440>, 2339839.34501022)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d176e210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d3b0e0>, 2339840.26594096)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d394ecd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3b711d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d39390>, 2339839.712915219)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3b71410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1c51ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3c02ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0181cc0>, 2339840.110058514)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1c539d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3bb6ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d20b4a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0272b30>, 2339840.233105764)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3a82e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d35a77d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d3a7b0>, 2339840.647170552)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d36826d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d395d150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d188fa80>, 2339841.3136489)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d395de90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3c40890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c88b0ffd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d38de0>, 2339840.030487857)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3c00dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d390f0>, 2339841.582648214)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3cbafd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c88bbe47c0>, 2339838.824727403)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c88b0fd950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d38750>, 2339840.547662442)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3c41e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3f6d790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3e17f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d02715c0>, 2339841.202587037)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3f6fc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a163c0>, 2339841.107838269)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8104c3cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3865190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a15080>, 2339840.981208887)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3c6cdd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d38a4dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110f8d70>, 2339841.267411256)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3abd150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810395050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d188e970>, 2339841.189521458)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8103969d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81081ef50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810243190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a14e50>, 2339841.31725586)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1855890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a15710>, 2339841.389792126)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810243890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d19a0490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2fa8fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a16510>, 2339841.89256494)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3e21950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a17a10>, 2339841.201396532)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810d68510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81015a610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a16820>, 2339841.919639868)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81015a8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c88b0e8410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a16900>, 2339841.825099239)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c88b0e83d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3bd5290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c88b0d35d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d01837e0>, 2339842.284315143)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d374d110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3809010>, 2339842.437036805)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c88b0d3550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810f26010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d380a2e0>, 2339842.415736129)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81069a6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c88c4596d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811c4b210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110f90f0>, 2339842.538418496)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d39a9ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110faf90>, 2339843.043305799)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811c49190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8105f2690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d380a510>, 2339842.973336236)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8111caa90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81079f5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0270440>, 2339843.064517489)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81079fdd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1777410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a14fa0>, 2339843.408116395)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3e16c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810e5e590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a16d60>, 2339843.388147871)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81019d710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810f4d450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81025c670>, 2339844.826493414)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3594d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811f0c350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81025d5c0>, 2339843.545857151)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8102d3bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810ddde50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d380b9a0>, 2339843.419165396)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2b69650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8117cd150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81025db00>, 2339845.855228883)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8117ce710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8112f4090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811827490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3f481a0>, 2339843.992784123)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81118a1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3f4ba10>, 2339844.341602754)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811827690>\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8113bcd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8113e3150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2db5550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81025d010>, 2339844.728464805)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811c61b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3603d90>, 2339844.721232743)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8113e0850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81025ef20>, 2339844.660170755)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2db4550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810edc150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811032650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81025fd90>, 2339844.135567721)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810ede090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81025f540>, 2339844.688263724)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811033450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81170e6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3f4b3f0>, 2339844.751118531)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81170fdd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8106bae10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3f4aac0>, 2339845.657818547)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8106bb610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7e46690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3f488a0>, 2339845.934476959)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7e44590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d32c4dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7e526d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3f48bb0>, 2339845.285014318)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d32c4890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7fc8a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7e02910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3f49390>, 2339845.67569738)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7fcbd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d22c63c0>, 2339845.734582392)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7e01950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7e9f0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7eda9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d21bcd00>, 2339845.58156459)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81160efd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d21bc210>, 2339845.721314396)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7edb610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7f70250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d3bcb0>, 2339846.564851007)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7f71f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7d36050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7ef8bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d275af10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2892b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d20be7b0>, 2339845.681302245)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7e50290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3f499b0>, 2339846.460639268)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7d35850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d28e36d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3f4b690>, 2339844.62019833)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d28e2d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8124cc600>, 2339847.771353263)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810dbdf90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8124ceeb0>, 2339847.568919967)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7d174d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3f4ae40>, 2339844.317153355)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2892a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7eadf50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ea4910>, 2339847.14522565)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7eac490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d01fadd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d01cef10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ea5080>, 2339847.213938501)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3c6f7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2781450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ea4d00>, 2339847.249058186)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7fa9d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7d4c550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3f486e0>, 2339847.876564307)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7d4c590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7f0cdd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ea52b0>, 2339847.495703311)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7e11e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0115c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7f63550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ea7930>, 2339848.531094626)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0114cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0173610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d029b050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1fc5a20>, 2339847.978914672)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d01737d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1fc7c40>, 2339847.984211099)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d02990d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d013ab50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0280f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811a3f700>, 2339848.046252438)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d013b410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811a3ea50>, 2339847.900340924)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0280b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d022d750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811a3e740>, 2339848.474841446)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7ed5910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7d340d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81025d9b0>, 2339847.4529354)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d01fac10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7d1f110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811cb5f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1fc66d0>, 2339847.358846235)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7f633d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0270ec0>, 2339848.430666958)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7d358d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8124cd780>, 2339846.454083277)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7d1c390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ea55c0>, 2339846.487423183)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811cb4290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d03dd250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ea74d0>, 2339848.563668027)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d03df450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81123b090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d02d0350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811a3d7f0>, 2339849.021888944)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7d1c290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811a3f3f0>, 2339850.173811919)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d02d2450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d06d4bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811a3c520>, 2339848.88094497)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7d1e910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d031f690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811a3d550>, 2339849.560189891)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d031fc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d04e9c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811a3c670>, 2339849.632724984)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d04eaed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d04cced0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1dab930>, 2339849.680097748)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d04cde90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7f47090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d05972d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1da9da0>, 2339849.992292958)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7f477d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1dab1c0>, 2339850.053603073)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0596e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d032f650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1da9a20>, 2339849.89989022)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d032ebd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0281090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1da81a0>, 2339849.673975114)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0280d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0295390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8124cd7f0>, 2339849.788438129)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7fed410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d05d8b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d043d490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810141470>, 2339851.28212414)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d05d8d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810143a80>, 2339851.351802944)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d043cc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d05c1d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0404810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810141fd0>, 2339850.240467806)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d049da50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8101430e0>, 2339850.172613159)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0406550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d053c450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2f32900>, 2339851.237382049)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d04d6c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0556490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d06fa850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810142a50>, 2339851.5328107)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d02dae10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7d1d0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810143cb0>, 2339850.925454832)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d03df1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d06b4d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d390ca60>, 2339851.322446759)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d06b4390>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d050fa90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d390d630>, 2339851.131311359)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d050c5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d03f91d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d024c510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811a3c1a0>, 2339852.553235204)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d068f2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d390f0e0>, 2339854.66979478)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d024f510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d067d910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d390c9f0>, 2339854.86810921)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d067e310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d06115d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d390d080>, 2339851.78220402)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0611d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0799c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0796150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8105575b0>, 2339851.92116064)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0683c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8105570e0>, 2339855.052204206)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0796310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0771f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811cb59d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811a3cf30>, 2339851.538176078)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d06fa0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810140bb0>, 2339850.044691839)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811cb4a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810143d90>, 2339865.096252159)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d05fb190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d074d390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810555550>, 2339854.95067173)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d074d190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0525410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810554c20>, 2339855.043286947)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0525050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d053fc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810556f20>, 2339864.794311953)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d053f490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0889290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d38f7d90>, 2339864.287089751)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d088b790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d08814d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d084dfd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d38f4e50>, 2339865.320210693)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0881790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d38f40c0>, 2339854.705641819)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d084cc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0898790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d09f4150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d38f7000>, 2339864.955229643)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d08983d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d38f54e0>, 2339865.098169311)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d09f7e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d09742d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d38f5da0>, 2339854.666958279)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d066d850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d06fad10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d08fbdd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d071c390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d068ae10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810555160>, 2339854.40543088)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d08f90d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d063ab90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d35016a0>, 2339854.9396226)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d094f0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1fc5710>, 2339854.877919213)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d071d850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810555a90>, 2339851.794714966)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0688890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3501a90>, 2339854.92246478)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0638850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0610290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d39940>, 2339854.661317399)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810d3b990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d09bb710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d38f4b40>, 2339877.431697075)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d09b9550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d08d5550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1fc40c0>, 2339864.737088278)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d081c910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d09d4c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3c2aba0>, 2339877.661775365)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d09d5990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0b40890>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3b298d0>, 2339879.173854441)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0b40b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0b190d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3b2b070>, 2339880.723016878)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0b1b3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0bc8850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3b2bd90>, 2339879.230965247)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0bed650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0c78b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d38f4f30>, 2339877.015069662)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0c7d210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0c4dad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0b0ef90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3b280c0>, 2339877.020741375)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0c544d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3b2b000>, 2339881.253579662)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0b0ce50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0dc3610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0da1690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1da8bb0>, 2339857.489964387)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0dc3950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3e49e80>, 2339877.20107614)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0da23d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0de3f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3e4b230>, 2339867.193562636)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0de0890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0e6c4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8104594e0>, 2339881.287399803)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0e75dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d11b7390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8116abfd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d088acd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d08adc50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810fa93d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81045ab30>, 2339880.718849467)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d11deb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81045aba0>, 2339880.464293986)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8116aa110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3501fd0>, 2339877.074767612)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d088ae90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3501390>, 2339877.08113913)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0a8d850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3e49cc0>, 2339880.192766652)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810fabf50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d09b8510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811366790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0ed2910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3501cc0>, 2339877.129136682)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d071e0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3e487c0>, 2339877.632906669)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d084c4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810458910>, 2339880.026842231)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0ed1a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0c4c8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3501320>, 2339881.877266586)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0c4c750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d11a2bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ae2580>, 2339880.246486836)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d11a1fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1a89c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ca3620>, 2339882.735015336)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1a8b3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8124377d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ca00c0>, 2339879.57740803)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c812435f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811d99d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ca1d30>, 2339881.247943759)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811869e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c88bf9c490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ca33f0>, 2339881.551302255)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c88bf9ca10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8117a6a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8125d8d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3f4bd20>, 2339881.154771589)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8117a72d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ca0360>, 2339881.755487109)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8125d92d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1c8d250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d11c8590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ca1ef0>, 2339880.802978062)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0b1af90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ca18d0>, 2339880.276384376)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d11c9d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811ad4810>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810142120>, 2339879.798492066)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811ad6a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8125dfe10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1a851d0>, 2339882.875381813)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0a8efd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811809910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ae0f30>, 2339882.351685856)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0a8ea10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c60b8650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81034aac0>, 2339884.082629225)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c60b8b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811b2c310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1a848a0>, 2339884.173332705)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811b2fc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c60bffd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8103482f0>, 2339882.4194583)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c60bfe90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6738f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810349160>, 2339884.477958202)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c67396d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8112d29d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81034a430>, 2339882.880013448)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810264bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c60f4fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5f0b710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81085a610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6810710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1a87930>, 2339882.32492243)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810858950>\n",
      "Evaluating workflow:  67%|██████▋   | 2021/3000 [06:34<02:19,  7.00it/s]Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1a86200>, 2339884.010221917)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0c15250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1a86c80>, 2339883.536373652)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5f0b510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810348c90>, 2339884.225213208)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6813610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c812434110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3b2a6d0>, 2339884.342035711)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c88bc5cf10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5f6fd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1a86f90>, 2339883.901983124)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5f6fbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5f526d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ca1be0>, 2339884.440095157)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5f51cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5f66e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c812417000>, 2339884.128731148)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5f0bd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811e8e350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5fb1a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c812416820>, 2339884.514079997)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81085a650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ca09f0>, 2339884.92424038)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5fb2050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5fd1a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8124166d0>, 2339884.632243473)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5fd23d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d11ecb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5e08590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d20bcde0>, 2339885.950494769)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0c30d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c88bc00bb0>, 2339885.358464573)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6618250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5fc62d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811917250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c60efc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5f1f310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5f504d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ae3d20>, 2339885.905231565)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0280650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811532850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c88bc01d30>, 2339886.253561817)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5fc6050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810459320>, 2339882.797560898)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811916490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d390dc50>, 2339884.639320516)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c60fae10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3b2bd20>, 2339884.305430168)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5f1f1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c88bc012b0>, 2339886.134649129)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5e27a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5f344d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810348a60>, 2339885.999682033)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0ea3190>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5e72e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81249da90>, 2339886.79315416)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1793c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5e91050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81249fa10>, 2339887.11294051)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5fd1090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5e9cb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ae2900>, 2339887.586116593)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5e9d210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0dc0790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81249eb30>, 2339886.893315152)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811809990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5ebe7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5ecfb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1a86430>, 2339887.310039819)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5ebea10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81249f0e0>, 2339886.078009917)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5ecfa10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5e7dbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5ef5f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810143d20>, 2339886.389969582)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5e07e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5d2a510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3c29e80>, 2339886.376869147)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5d2a3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5d547d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c812415400>, 2339886.952383599)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5eb5250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5ffab10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81182e740>, 2339887.137773365)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0ea30d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81182cc20>, 2339886.472956955)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5ef5e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5d0de50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81182cfa0>, 2339887.494426783)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5d0d950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5d05c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81249d470>, 2339887.665157937)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5d05b50>\n",
      "Evaluating workflow:  67%|██████▋   | 2024/3000 [06:34<01:46,  9.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2027/3000 [06:34<01:24, 11.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2029/3000 [06:34<01:43,  9.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2031/3000 [06:35<02:00,  8.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2035/3000 [06:36<02:39,  6.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2039/3000 [06:36<01:44,  9.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2026-01-13 12:49:13.335\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:49:13.336\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2041/3000 [06:39<07:00,  2.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:49:13.365\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:49:13.388\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:49:13.634\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2045/3000 [06:39<04:38,  3.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:49:14.818\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert analyst of perturbation datasets. For K562 cells subjected to perturbation of SEC61A1, does the expression profile of IL18R1 indicate a significant change relative to control conditions? Reply strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are an expert analyst of perturbation datasets. For K562 cells subjected to perturbation of SEC61A1, does the expression profile of IL18R1 indicate a significant change relative to control conditions? Reply strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 357}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 8 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:49:14.966\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in single-cell biology and functional genomics. In K562 cells, SEC61B is perturbed and the expression of PSMC1 is measured. Does this perturbation cause a significant change in PSMC1 expression? Answer strictly in the format 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are an expert in single-cell biology and functional genomics. In K562 cells, SEC61B is perturbed and the expression of PSMC1 is measured. Does this perturbation cause a significant change in PSMC1 expression? Answer strictly in the format 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 2820}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 10 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2047/3000 [06:41<05:57,  2.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:49:15.029\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a genomics expert evaluating perturbation experiments. In K562 cells, ASCC3 is perturbed and the expression of HPCAL1 is measured. Determine whether HPCAL1 shows a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are a genomics expert evaluating perturbation experiments. In K562 cells, ASCC3 is perturbed and the expression of HPCAL1 is measured. Determine whether HPCAL1 shows a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 2976}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 10 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:49:15.234\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a genomics expert evaluating perturbation experiments. In K562 cells, ZNF326 is perturbed and the expression of PHF3 is measured. Determine whether PHF3 shows a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are a genomics expert evaluating perturbation experiments. In K562 cells, ZNF326 is perturbed and the expression of PHF3 is measured. Determine whether PHF3 shows a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 86}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 10 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2049/3000 [06:41<05:01,  3.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:49:15.279\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to KCTD16 and then measure expression of CCDC69. Does this perturbation cause a significant change in CCDC69 expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to KCTD16 and then measure expression of CCDC69. Does this perturbation cause a significant change in CCDC69 expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 3444}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 10 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:49:15.317\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to SEC61G and then measure expression of MTPN. Does this perturbation cause a significant change in MTPN expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to SEC61G and then measure expression of MTPN. Does this perturbation cause a significant change in MTPN expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 4734}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 10 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:49:15.322\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to MRGBP and then measure expression of ARHGAP30. Does this perturbation cause a significant change in ARHGAP30 expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to MRGBP and then measure expression of ARHGAP30. Does this perturbation cause a significant change in ARHGAP30 expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 1474}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 9 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:49:15.393\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a domain expert in functional genomics. For experiments carried out in K562 cells, we perturb TTI1 and examine the expression of RP11-16E12.1. Does perturbing TTI1 lead to a significant change in RP11-16E12.1 expression? Reply only with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are a domain expert in functional genomics. For experiments carried out in K562 cells, we perturb TTI1 and examine the expression of RP11-16E12.1. Does perturbing TTI1 lead to a significant change in RP11-16E12.1 expression? Reply only with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 3231}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 9 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2053/3000 [06:41<03:12,  4.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:49:15.465\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in gene expression analysis. For K562 cells, assess whether perturbation of PPWD1 is associated with a significant change in CENPC expression compared with unperturbed controls. Answer strictly as 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are an expert in gene expression analysis. For K562 cells, assess whether perturbation of PPWD1 is associated with a significant change in CENPC expression compared with unperturbed controls. Answer strictly as 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 4455}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 9 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:49:15.493\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a genomics expert evaluating perturbation experiments. In K562 cells, OST4 is perturbed and the expression of UHRF1BP1L is measured. Determine whether UHRF1BP1L shows a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are a genomics expert evaluating perturbation experiments. In K562 cells, OST4 is perturbed and the expression of UHRF1BP1L is measured. Determine whether UHRF1BP1L shows a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 246}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 9 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:49:15.533\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are assisting with the interpretation of perturbation-based expression data. In K562 cells, HARS is perturbed and SAMM50 expression is quantified. Does this perturbation result in a significant change in SAMM50 expression compared with control cells? Respond exactly as 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are assisting with the interpretation of perturbation-based expression data. In K562 cells, HARS is perturbed and SAMM50 expression is quantified. Does this perturbation result in a significant change in SAMM50 expression compared with control cells? Respond exactly as 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 622}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 9 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▊   | 2056/3000 [06:41<02:26,  6.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:49:15.571\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 7 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:49:22.477\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:49:22.482\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▊   | 2058/3000 [06:48<14:09,  1.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:49:22.507\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:49:22.523\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:49:24.556\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 10 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▊   | 2062/3000 [06:50<11:45,  1.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:49:24.593\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 10 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:49:24.657\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 10 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2064/3000 [06:50<09:20,  1.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:49:24.678\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 10 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:49:24.702\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 10 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:49:24.952\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 10 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2067/3000 [06:51<06:49,  2.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:49:24.962\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are assisting with the interpretation of perturbation-based expression data. In K562 cells, EIF2B4 is perturbed and RP11-90P5.2 expression is quantified. Does this perturbation result in a significant change in RP11-90P5.2 expression compared with control cells? Respond exactly as 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are assisting with the interpretation of perturbation-based expression data. In K562 cells, EIF2B4 is perturbed and RP11-90P5.2 expression is quantified. Does this perturbation result in a significant change in RP11-90P5.2 expression compared with control cells? Respond exactly as 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 402}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 10 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:49:25.370\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 10 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2069/3000 [06:51<05:58,  2.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:49:34.631\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2070/3000 [07:00<25:36,  1.65s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:49:34.652\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:49:34.668\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a genomics expert evaluating perturbation experiments. In K562 cells, SAMM50 is perturbed and the expression of NUF2 is measured. Determine whether NUF2 shows a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are a genomics expert evaluating perturbation experiments. In K562 cells, SAMM50 is perturbed and the expression of NUF2 is measured. Determine whether NUF2 shows a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 1396}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:49:34.680\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2074/3000 [07:00<14:17,  1.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:49:35.010\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to SRP72 and then measure expression of NOX5. Does this perturbation cause a significant change in NOX5 expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to SRP72 and then measure expression of NOX5. Does this perturbation cause a significant change in NOX5 expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 3414}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2076/3000 [07:01<11:14,  1.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:49:35.650\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:49:35.697\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2078/3000 [07:01<09:41,  1.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:49:35.776\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in gene regulation studies. For experiments performed in K562 cells, YIPF5 is perturbed and CTD-2001C12.1 expression is measured. Determine whether CTD-2001C12.1 exhibits a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are an expert in gene regulation studies. For experiments performed in K562 cells, YIPF5 is perturbed and CTD-2001C12.1 expression is measured. Determine whether CTD-2001C12.1 exhibits a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 579}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:49:35.792\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a genomics expert evaluating perturbation experiments. In K562 cells, IER3IP1 is perturbed and the expression of DDX6 is measured. Determine whether DDX6 shows a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are a genomics expert evaluating perturbation experiments. In K562 cells, IER3IP1 is perturbed and the expression of DDX6 is measured. Determine whether DDX6 shows a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 526}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:49:35.846\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2081/3000 [07:01<06:29,  2.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:49:36.235\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a domain expert in functional genomics. For experiments carried out in K562 cells, we perturb MRPL39 and examine the expression of RP13-216E22.4. Does perturbing MRPL39 lead to a significant change in RP13-216E22.4 expression? Reply only with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are a domain expert in functional genomics. For experiments carried out in K562 cells, we perturb MRPL39 and examine the expression of RP13-216E22.4. Does perturbing MRPL39 lead to a significant change in RP13-216E22.4 expression? Reply only with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 2431}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2083/3000 [07:03<06:59,  2.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2026-01-13 12:49:36.966\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 2085/3000 [07:03<05:45,  2.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|██████▉   | 2088/3000 [07:04<04:35,  3.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|██████▉   | 2092/3000 [07:04<02:42,  5.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 2094/3000 [07:05<03:17,  4.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|███████   | 2100/3000 [07:05<02:15,  6.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 2102/3000 [07:05<01:57,  7.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 2104/3000 [07:06<02:34,  5.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 2107/3000 [07:06<02:06,  7.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|███████   | 2110/3000 [07:07<03:05,  4.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 2111/3000 [07:07<02:49,  5.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 2113/3000 [07:08<02:40,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 2119/3000 [07:08<01:39,  8.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2122/3000 [07:08<01:32,  9.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 2128/3000 [07:10<01:54,  7.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 2132/3000 [07:10<01:41,  8.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2136/3000 [07:10<01:17, 11.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████▏  | 2140/3000 [07:11<02:12,  6.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2145/3000 [07:12<01:40,  8.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2148/3000 [07:12<01:17, 11.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2150/3000 [07:12<01:18, 10.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2152/3000 [07:12<01:24, 10.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2154/3000 [07:13<01:25,  9.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2157/3000 [07:13<02:00,  6.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2158/3000 [07:14<03:03,  4.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2162/3000 [07:14<02:02,  6.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2169/3000 [07:14<01:07, 12.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2171/3000 [07:15<01:25,  9.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2176/3000 [07:15<01:31,  8.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 2180/3000 [07:16<01:43,  7.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2184/3000 [07:16<01:26,  9.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2187/3000 [07:17<01:17, 10.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2190/3000 [07:17<01:41,  8.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2191/3000 [07:18<02:08,  6.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2192/3000 [07:18<02:18,  5.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 2201/3000 [07:18<01:17, 10.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▎  | 2205/3000 [07:19<01:04, 12.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▎  | 2207/3000 [07:19<01:37,  8.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▎  | 2209/3000 [07:19<01:33,  8.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▎  | 2212/3000 [07:20<01:41,  7.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2214/3000 [07:20<01:22,  9.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 2218/3000 [07:20<01:20,  9.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2220/3000 [07:21<01:39,  7.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2224/3000 [07:21<01:11, 10.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2226/3000 [07:21<01:50,  7.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2227/3000 [07:22<02:06,  6.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 2232/3000 [07:22<01:26,  8.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2235/3000 [07:22<01:05, 11.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2237/3000 [07:23<01:15, 10.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2240/3000 [07:23<01:07, 11.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2242/3000 [07:23<01:17,  9.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2244/3000 [07:23<01:19,  9.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2246/3000 [07:24<01:25,  8.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2248/3000 [07:24<01:47,  7.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▌  | 2255/3000 [07:24<00:57, 12.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 2257/3000 [07:24<00:56, 13.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 2259/3000 [07:25<01:31,  8.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 2261/3000 [07:25<01:42,  7.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 2263/3000 [07:25<01:26,  8.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2265/3000 [07:26<01:28,  8.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 2269/3000 [07:26<01:24,  8.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2271/3000 [07:26<01:13,  9.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 2276/3000 [07:27<01:17,  9.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2278/3000 [07:28<01:56,  6.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2280/3000 [07:28<02:13,  5.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 2286/3000 [07:28<01:15,  9.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▋  | 2289/3000 [07:29<01:14,  9.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▋  | 2291/3000 [07:29<01:18,  9.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▋  | 2293/3000 [07:30<01:59,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 2298/3000 [07:30<01:31,  7.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 2302/3000 [07:30<01:12,  9.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2304/3000 [07:31<01:11,  9.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2309/3000 [07:31<01:05, 10.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 2312/3000 [07:32<01:49,  6.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 2317/3000 [07:32<01:06, 10.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2319/3000 [07:32<00:59, 11.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2321/3000 [07:33<01:35,  7.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 2324/3000 [07:34<01:57,  5.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2326/3000 [07:34<02:21,  4.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2331/3000 [07:35<01:26,  7.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2333/3000 [07:35<01:23,  7.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2337/3000 [07:35<01:15,  8.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2340/3000 [07:36<01:41,  6.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2342/3000 [07:36<01:45,  6.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:50:13.526\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are assisting with the interpretation of perturbation-based expression data. In K562 cells, PDIA6 is perturbed and DOCK8 expression is quantified. Does this perturbation result in a significant change in DOCK8 expression compared with control cells? Respond exactly as 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are assisting with the interpretation of perturbation-based expression data. In K562 cells, PDIA6 is perturbed and DOCK8 expression is quantified. Does this perturbation result in a significant change in DOCK8 expression compared with control cells? Respond exactly as 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 1562}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 11 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2344/3000 [07:39<06:06,  1.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:50:13.603\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to IDH3A and then measure expression of RP11-334C17.5. Does this perturbation cause a significant change in RP11-334C17.5 expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to IDH3A and then measure expression of RP11-334C17.5. Does this perturbation cause a significant change in RP11-334C17.5 expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 1004}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 11 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2346/3000 [07:39<04:44,  2.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:50:13.997\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 11 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2348/3000 [07:40<03:30,  3.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2349/3000 [07:42<06:38,  1.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:50:24.581\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in regulatory genomics. Consider data from K562 cells in which SOCS1 is perturbed and DDX3X expression is observed. Does this perturbation lead to a significant difference in DDX3X expression relative to control conditions? Answer strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are an expert in regulatory genomics. Consider data from K562 cells in which SOCS1 is perturbed and DDX3X expression is observed. Does this perturbation lead to a significant difference in DDX3X expression relative to control conditions? Answer strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 1083}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 10 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2350/3000 [07:50<24:33,  2.27s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:50:24.617\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in gene expression analysis. For K562 cells, assess whether perturbation of GMPPB is associated with a significant change in ATP2A3 expression compared with unperturbed controls. Answer strictly as 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are an expert in gene expression analysis. For K562 cells, assess whether perturbation of GMPPB is associated with a significant change in ATP2A3 expression compared with unperturbed controls. Answer strictly as 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 1835}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 10 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:50:24.669\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a genomics expert evaluating perturbation experiments. In K562 cells, MRGBP is perturbed and the expression of AC079466.1 is measured. Determine whether AC079466.1 shows a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are a genomics expert evaluating perturbation experiments. In K562 cells, MRGBP is perturbed and the expression of AC079466.1 is measured. Determine whether AC079466.1 shows a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 1446}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 10 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:50:24.681\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a domain expert in functional genomics. For experiments carried out in K562 cells, we perturb MARS and examine the expression of RP11-685N10.1. Does perturbing MARS lead to a significant change in RP11-685N10.1 expression? Reply only with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are a domain expert in functional genomics. For experiments carried out in K562 cells, we perturb MARS and examine the expression of RP11-685N10.1. Does perturbing MARS lead to a significant change in RP11-685N10.1 expression? Reply only with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 2751}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 10 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2353/3000 [07:50<13:09,  1.22s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:50:24.694\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a genomics expert evaluating perturbation experiments. In K562 cells, OST4 is perturbed and the expression of DOK3 is measured. Determine whether DOK3 shows a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are a genomics expert evaluating perturbation experiments. In K562 cells, OST4 is perturbed and the expression of DOK3 is measured. Determine whether DOK3 shows a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 266}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 10 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:50:24.748\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in single-cell biology and functional genomics. In K562 cells, COPZ1 is perturbed and the expression of LINC00862 is measured. Does this perturbation cause a significant change in LINC00862 expression? Answer strictly in the format 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are an expert in single-cell biology and functional genomics. In K562 cells, COPZ1 is perturbed and the expression of LINC00862 is measured. Does this perturbation cause a significant change in LINC00862 expression? Answer strictly in the format 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 4590}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 10 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:50:24.827\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert analyst of perturbation datasets. For K562 cells subjected to perturbation of TARS, does the expression profile of AC007038.7 indicate a significant change relative to control conditions? Reply strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are an expert analyst of perturbation datasets. For K562 cells subjected to perturbation of TARS, does the expression profile of AC007038.7 indicate a significant change relative to control conditions? Reply strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 2927}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 10 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▊  | 2356/3000 [07:50<08:03,  1.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:50:24.837\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in gene expression analysis. For K562 cells, assess whether perturbation of SEC61A1 is associated with a significant change in RP11-423O2.5 expression compared with unperturbed controls. Answer strictly as 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are an expert in gene expression analysis. For K562 cells, assess whether perturbation of SEC61A1 is associated with a significant change in RP11-423O2.5 expression compared with unperturbed controls. Answer strictly as 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 305}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 10 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:50:24.971\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in gene regulation studies. For experiments performed in K562 cells, PTDSS1 is perturbed and RNF19A expression is measured. Determine whether RNF19A exhibits a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are an expert in gene regulation studies. For experiments performed in K562 cells, PTDSS1 is perturbed and RNF19A expression is measured. Determine whether RNF19A exhibits a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 3769}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 10 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▊  | 2358/3000 [07:51<06:01,  1.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:50:25.049\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert analyst of perturbation datasets. For K562 cells subjected to perturbation of NEDD8, does the expression profile of GPRC5C indicate a significant change relative to control conditions? Reply strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are an expert analyst of perturbation datasets. For K562 cells subjected to perturbation of NEDD8, does the expression profile of GPRC5C indicate a significant change relative to control conditions? Reply strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 2227}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 10 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:50:25.224\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert analyst of perturbation datasets. For K562 cells subjected to perturbation of DERL2, does the expression profile of CST7 indicate a significant change relative to control conditions? Reply strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are an expert analyst of perturbation datasets. For K562 cells subjected to perturbation of DERL2, does the expression profile of CST7 indicate a significant change relative to control conditions? Reply strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 2207}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 10 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▊  | 2360/3000 [07:51<04:41,  2.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:50:25.243\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in gene expression analysis. For K562 cells, assess whether perturbation of QARS is associated with a significant change in ITGB2 expression compared with unperturbed controls. Answer strictly as 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are an expert in gene expression analysis. For K562 cells, assess whether perturbation of QARS is associated with a significant change in ITGB2 expression compared with unperturbed controls. Answer strictly as 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 1305}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 10 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:50:25.257\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a genomics expert evaluating perturbation experiments. In K562 cells, AMIGO3 is perturbed and the expression of RSL24D1 is measured. Determine whether RSL24D1 shows a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are a genomics expert evaluating perturbation experiments. In K562 cells, AMIGO3 is perturbed and the expression of RSL24D1 is measured. Determine whether RSL24D1 shows a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 876}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 10 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:50:25.262\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert analyst of perturbation datasets. For K562 cells subjected to perturbation of SRPR, does the expression profile of CD9 indicate a significant change relative to control conditions? Reply strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are an expert analyst of perturbation datasets. For K562 cells subjected to perturbation of SRPR, does the expression profile of CD9 indicate a significant change relative to control conditions? Reply strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 427}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 10 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2364/3000 [07:53<05:39,  1.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2026-01-13 12:50:35.141\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2365/3000 [08:01<15:32,  1.47s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:50:35.282\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2366/3000 [08:01<13:12,  1.25s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:50:35.398\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to AMIGO3 and then measure expression of SNHG12. Does this perturbation cause a significant change in SNHG12 expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to AMIGO3 and then measure expression of SNHG12. Does this perturbation cause a significant change in SNHG12 expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 844}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2367/3000 [08:01<10:54,  1.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:50:35.667\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a genomics expert evaluating perturbation experiments. In K562 cells, SEC63 is perturbed and the expression of MORF4L2 is measured. Determine whether MORF4L2 shows a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are a genomics expert evaluating perturbation experiments. In K562 cells, SEC63 is perturbed and the expression of MORF4L2 is measured. Determine whether MORF4L2 shows a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 2626}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2368/3000 [08:01<09:10,  1.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:50:35.825\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2369/3000 [08:01<07:24,  1.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2370/3000 [08:03<09:14,  1.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2372/3000 [08:03<05:50,  1.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2026-01-13 12:50:37.530\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 2376/3000 [08:04<03:29,  2.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2026-01-13 12:50:38.620\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a functional genomics specialist. In K562 cells, we perturb YIPF5 and monitor PPP1R16B expression. Decide whether this perturbation leads to a significant alteration in PPP1R16B expression. Answer only in the format 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are a functional genomics specialist. In K562 cells, we perturb YIPF5 and monitor PPP1R16B expression. Decide whether this perturbation leads to a significant alteration in PPP1R16B expression. Answer only in the format 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 598}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2377/3000 [08:04<03:32,  2.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2380/3000 [08:04<02:14,  4.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2382/3000 [08:05<02:06,  4.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|███████▉  | 2385/3000 [08:06<02:25,  4.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 2389/3000 [08:06<01:23,  7.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 2391/3000 [08:06<01:14,  8.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 2394/3000 [08:07<01:22,  7.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 2397/3000 [08:07<01:50,  5.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 2400/3000 [08:08<01:40,  5.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2404/3000 [08:08<01:01,  9.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2406/3000 [08:08<01:07,  8.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2408/3000 [08:09<01:12,  8.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2410/3000 [08:09<01:36,  6.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 2414/3000 [08:10<01:28,  6.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2416/3000 [08:10<01:22,  7.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2420/3000 [08:10<01:03,  9.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2425/3000 [08:10<00:46, 12.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2427/3000 [08:11<01:39,  5.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2431/3000 [08:12<01:23,  6.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2433/3000 [08:12<01:11,  7.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2435/3000 [08:12<01:10,  7.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████▏ | 2441/3000 [08:13<00:47, 11.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████▏ | 2443/3000 [08:13<01:34,  5.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2447/3000 [08:14<01:19,  6.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2453/3000 [08:15<00:57,  9.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2455/3000 [08:15<00:50, 10.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2457/3000 [08:15<01:01,  8.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2460/3000 [08:15<00:49, 10.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2463/3000 [08:16<01:24,  6.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2465/3000 [08:16<01:17,  6.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2468/3000 [08:17<01:15,  7.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2470/3000 [08:17<01:00,  8.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2474/3000 [08:17<00:58,  9.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2476/3000 [08:17<00:49, 10.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2481/3000 [08:18<00:44, 11.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2486/3000 [08:19<01:18,  6.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2490/3000 [08:19<00:55,  9.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2492/3000 [08:19<00:49, 10.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2497/3000 [08:20<00:59,  8.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2499/3000 [08:20<01:13,  6.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2501/3000 [08:21<01:17,  6.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▎ | 2507/3000 [08:21<00:48, 10.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▎ | 2512/3000 [08:22<00:44, 10.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2514/3000 [08:22<00:59,  8.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2516/3000 [08:22<01:08,  7.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2519/3000 [08:23<01:08,  7.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2524/3000 [08:23<00:42, 11.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2526/3000 [08:23<00:37, 12.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2528/3000 [08:23<00:39, 12.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2530/3000 [08:24<00:52,  8.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2532/3000 [08:24<01:13,  6.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2535/3000 [08:25<01:18,  5.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▍ | 2538/3000 [08:25<01:09,  6.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2540/3000 [08:25<00:55,  8.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2543/3000 [08:26<00:52,  8.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▍ | 2548/3000 [08:26<00:41, 10.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▌ | 2552/3000 [08:27<00:44, 10.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2554/3000 [08:27<00:54,  8.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2556/3000 [08:27<00:49,  9.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2558/3000 [08:27<00:57,  7.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2562/3000 [08:28<00:42, 10.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2564/3000 [08:28<00:42, 10.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2567/3000 [08:28<00:51,  8.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5da5150>\n",
      "Evaluating workflow:  86%|████████▌ | 2568/3000 [08:29<01:21,  5.32it/s]Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3c2ae40>, 2339888.898729499)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5e7d890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5c946d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3c2ba10>, 2339889.748115402)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5c95810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5b6d990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8113146e0>, 2339892.143964711)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5b6dcd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5db3950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811315d30>, 2339890.609164668)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5db2dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5b08710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811629390>, 2339889.260057752)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5b08510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5a1dc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811314ec0>, 2339891.218204869)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5da6210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c59a6f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8111ed400>, 2339893.810908638)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5ade210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c590aa50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c59453d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5968350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81249dc50>, 2339893.731016813)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c59089d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3d45860>, 2339893.885262671)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5945ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3d44f30>, 2339893.857858542)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5968890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c584afd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811205780>, 2339895.610906897)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c584b1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c58de990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8111ee890>, 2339895.712761541)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c58de850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5a3d590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810d755c0>, 2339896.375171454)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c59b2010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5a91790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5969450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3b2a660>, 2339895.82942497)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5a91c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810ec42f0>, 2339894.506323899)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5969650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c575a010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810d76c10>, 2339897.000850104)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c575a2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5630c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810676740>, 2339899.348780279)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5630d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5637210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8106756a0>, 2339898.045781812)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5637310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c578cad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c57d9250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c56d26d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810d762e0>, 2339897.645780285)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5acd250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810d74c90>, 2339896.210657846)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c57d9110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3d44ad0>, 2339900.19419367)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c56d3ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c56db310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3d47e70>, 2339899.442868273)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c56db1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c56c3190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810746ac0>, 2339898.847765896)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c56a3d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c56b9850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810745400>, 2339899.179775458)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c56b9fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c556a750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810833070>, 2339899.684904204)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c556a650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c56f5ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5518e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8107456a0>, 2339899.290332613)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c56f5e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811204050>, 2339899.487183578)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5519510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c54781d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c7f77ee0>, 2339901.801247757)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5479310>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c54be550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c7f77460>, 2339901.535186138)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c54bd790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5684d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8108315c0>, 2339899.797161847)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5631050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5370490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c7f75940>, 2339901.499704736)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5335750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c53af450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81193b7e0>, 2339903.033554061)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c55ed7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c53af050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d034a200>, 2339902.749661028)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c53af910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c536a010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c7f77bd0>, 2339902.853299402)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5368c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c521fad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c67c9c50>, 2339905.302578047)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c521f950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c52dc4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c67c9e10>, 2339905.384018283)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c56063d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5123e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c5171630>, 2339906.272466054)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5123a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c51a1d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c51718d0>, 2339905.12176263)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c51a3990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c52a5710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5181010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c67c9080>, 2339903.716902405)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c52a5510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c5170a60>, 2339906.162138174)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5180ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c50e9790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4f3cf10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c5173a10>, 2339907.519866758)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4f3cdd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4f03090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c5173b60>, 2339908.200732267)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4f02b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4f6a3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c5171390>, 2339907.871530942)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4f6b150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c51ef090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c51709f0>, 2339908.285893742)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5122950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c50288d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4feac90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5063dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4e01d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c4f2c280>, 2339907.30575206)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5001590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c5053150>, 2339910.535787425)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c50d44d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810747c40>, 2339910.529790919)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5063890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c4f2f620>, 2339910.805680573)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c50f2810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4eb0710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c4e3e040>, 2339912.597222285)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4e00750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4e6f210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c5170210>, 2339909.933543024)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4e6f110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4ef84d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c4d0c0c0>, 2339910.951571757)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4e94990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4ed1e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810d75630>, 2339910.924956794)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4ed25d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5046fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4d545d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c4f2d470>, 2339911.03144998)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4ee3810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c4d0cec0>, 2339910.791654972)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4d543d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4eeaed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c5051630>, 2339910.910345348)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4eeb250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4d6b310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4d8ab10>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c4f2fcb0>, 2339911.369471634)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4d6b210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c4d0db70>, 2339910.899085578)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4d8a5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4d9a6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c4d0e040>, 2339911.724397961)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4d9a350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4d16a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4dbcd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c4d0e270>, 2339911.018105059)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4d17910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8119387c0>, 2339911.381423778)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4dbd5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4e91ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4d31550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c51734d0>, 2339910.525698458)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4d31350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4c0fd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5f9eb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5d7b2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5db2250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5d8a890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5dc8810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5dcf910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5deb910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5c12fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5f95690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5de8c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5c69510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5c3b890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5c79a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5c56510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5ca26d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5ccb790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5cf9ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5b2c990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5ddc810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5b64ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5b0b210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5b88bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5ba8490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5b26650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5bcba90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5a00290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5b6d150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5a10990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5a2b950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5b25050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5a4f290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5ab8510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5ddc510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5acca90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5ad5250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5adfa10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5af4ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5a488d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5a93e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c590b150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5909450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c598a6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c59995d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c59c1990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c59e4bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c599b050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5827ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5a014d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c59c07d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5816dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c589a6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c58c5290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c58dc850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c57098d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c58b14d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c573a6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c59a6cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c576f850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5738590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c59c0210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5853890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c57a90d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c57fc590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5637d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c561f550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5668590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5633bd0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5686410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5693a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5562c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5543350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5593d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c55571d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c569cb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c55dfed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c55e6c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c55dcd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5419b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c542f850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5456790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c54635d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c547bf10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c54a9d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c549cd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c55a4390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c54ce990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c54e0e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5308310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c55fce50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5343610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5356c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5389510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c53d6d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c53dd290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c53f2c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c520ad90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c541a190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c524d690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c527b7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c53d5c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c52898d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c510b1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c525c110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5158650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c518e410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c51d44d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c525da50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c526b150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c51485d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c55d5090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c51f82d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c51fb0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5046f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5244f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c50637d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c507d110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5095c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c50b9b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4f014d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4f1e6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d034af90>, 2339908.374047126)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c50e9890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5054750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c50d7210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c50f26d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4f44d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c528add0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c50ec110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c509f1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4e67310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4e80b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4f9ef10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4ec7a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c5051b70>, 2339910.519029091)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4e671d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c50504b0>, 2339912.394785684)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4d32990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c4d0ff50>, 2339912.619551149)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4c0fbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81182dfd0>, 2339887.415996362)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5e0b810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81182def0>, 2339888.003862601)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5d78950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81172b5b0>, 2339888.051996626)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5db2910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81182e900>, 2339888.369855537)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5d8a7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811728ad0>, 2339888.793444991)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5dc8650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ae1e10>, 2339888.124785849)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5dcfd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81172a970>, 2339888.33820018)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5deb710>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8117292b0>, 2339888.442296463)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5c12e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8117299b0>, 2339888.088255633)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5c15090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811729780>, 2339888.330959344)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5d5aed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8117280c0>, 2339889.052328692)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5c693d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81172ba80>, 2339889.234770785)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5c3bf90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811728bb0>, 2339889.213909145)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5c79d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81172baf0>, 2339888.789170718)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5c56a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81182e2e0>, 2339889.071686217)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5ca21d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811628a60>, 2339890.170979131)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5ccbe50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811728c90>, 2339889.964198014)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5cf9fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81182c6e0>, 2339889.725106567)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5de9190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811628c90>, 2339890.216227461)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5ddc650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811316660>, 2339890.027167433)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5b64e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81172b850>, 2339890.077566494)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5b0ab50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81249f930>, 2339890.774317378)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5b89050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8113160b0>, 2339889.93299769)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5ba87d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811729b00>, 2339890.847975835)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5e7f0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8113164a0>, 2339890.781446729)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5bcbf50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811314d00>, 2339890.823253456)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5dcd850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811316a50>, 2339891.507005956)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5b2c490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811317930>, 2339891.433171168)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5a10790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811205080>, 2339891.690167389)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5a29090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811204360>, 2339891.771333271)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5a542d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8116299b0>, 2339891.735551496)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5a4f910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8112055c0>, 2339892.23482615)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5aafb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811205da0>, 2339892.297954201)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5a5e950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811628520>, 2339894.31185443)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5b2cfd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8112052b0>, 2339892.481494186)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5a722d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811204d00>, 2339894.154036508)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5adf8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3d452b0>, 2339892.12135143)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5e1a1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811207a10>, 2339892.822704406)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5c5ae10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811204980>, 2339892.654609759)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5a93d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811315550>, 2339892.649837456)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c590ad50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3d46b30>, 2339893.48093359)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c590a890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811206c80>, 2339893.446390626)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5db3290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8111ee3c0>, 2339893.337068903)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5999490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8111ee040>, 2339894.206417822)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c59c1850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8111edef0>, 2339893.937852546)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c59e5290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8111ecec0>, 2339894.564598109)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5d0e550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8111efcb0>, 2339894.604166039)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5827dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3d45550>, 2339894.654842548)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c59c34d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81249e7b0>, 2339894.24392406)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5dc8d90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8111ec520>, 2339894.490179358)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5816c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3d44440>, 2339895.05656521)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c589a690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810ec52b0>, 2339896.371105589)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5cfa290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81034b070>, 2339895.618381576)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c58dc650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810d75390>, 2339895.654134206)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5709790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810d76ba0>, 2339895.719872911)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5854550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810d75160>, 2339895.878593561)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c573bb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811204590>, 2339896.406760786)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c59a7290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810d77770>, 2339896.250807692)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c576f710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810d74670>, 2339895.754920572)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5898810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810d76900>, 2339897.060686213)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c59c0b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811206120>, 2339896.359825014)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5af7650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8111ecc20>, 2339896.946259046)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c57a9310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810675940>, 2339897.484597506)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c57fcc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810676e40>, 2339898.002056375)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5784450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810674de0>, 2339897.418719243)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c588bf10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810675400>, 2339897.961526166)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5708b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810674440>, 2339897.702461193)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5632cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810675be0>, 2339898.413447643)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5686210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810ec7e00>, 2339897.838868222)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5693d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810ec6740>, 2339898.676362054)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c552b310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810832eb0>, 2339899.184427634)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5541590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8108330e0>, 2339899.858894926)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5593bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810833380>, 2339899.247644997)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5557050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810d76510>, 2339899.982806534)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c569cf50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810832660>, 2339899.452351565)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c55dfd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810ec57f0>, 2339900.956078738)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c55e4650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810830750>, 2339900.400351376)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c55dcb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81193a350>, 2339899.977608531)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5419a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81193bf50>, 2339900.765542785)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c542f710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811938050>, 2339900.708511576)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c54546d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8119388a0>, 2339900.691440277)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5463210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810d76270>, 2339900.94618175)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c547bdd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811939470>, 2339901.639466661)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c54608d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81193b4d0>, 2339901.210351008)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c549cb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81193ae40>, 2339902.902638199)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c55a4c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811938210>, 2339900.752954814)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c54cf050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c7f74210>, 2339901.223646182)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5582f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c7f77cb0>, 2339901.80656859)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5308810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810d747c0>, 2339901.564845412)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5674910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811939e10>, 2339901.594136115)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5341190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8108334d0>, 2339902.178610479)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5357110>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c7f75860>, 2339902.108828306)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c53893d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d034a9e0>, 2339903.147359823)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c53d74d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d034a820>, 2339903.015149937)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c53dce50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d034a970>, 2339902.694079596)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c53f0450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d034a270>, 2339905.417816711)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c520abd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810747bd0>, 2339902.604872942)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5229790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810833620>, 2339903.035079565)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c524d490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810744c90>, 2339904.180297019)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c527b690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0348c90>, 2339904.2325553)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c53d5b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0348e50>, 2339903.809051493)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5289710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c67ca890>, 2339905.521319604)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5108990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c67cb770>, 2339904.77731564)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5489810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c67c89f0>, 2339905.62099737)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c514bbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c5170f30>, 2339905.407187452)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c517a6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c51720b0>, 2339905.154915961)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c51d5d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c67c8d00>, 2339904.400979879)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c525d950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c67c9630>, 2339904.84209682)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c526b590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81193bb60>, 2339904.261252066)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5148c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c5171780>, 2339905.025236773)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c55d6c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c5172900>, 2339906.779407549)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c51f8b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8111ecf30>, 2339906.260648147)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c51fb410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c5050280>, 2339906.337109185)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5046bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c5050670>, 2339906.633884622)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c52dd850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8106775b0>, 2339906.855123654)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5063690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0349240>, 2339906.653170779)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c507fad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c5051470>, 2339908.105725545)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c52a6350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c5051f60>, 2339908.114571155)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c50baf50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c50539a0>, 2339908.141136286)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4f01290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c5053e70>, 2339907.635246649)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4f1d210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c5051010>, 2339906.834315617)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c50bbf90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c5052890>, 2339906.620584651)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c50d70d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8108310f0>, 2339907.022872745)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c50f1690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810831710>, 2339907.662868824)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4f452d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c4f2d860>, 2339908.764700272)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c528a790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c4e3c670>, 2339909.131853383)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c50ef0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c4e3c980>, 2339908.70471914)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4f85c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c4e3d320>, 2339909.659719996)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c50f0b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c5050b40>, 2339908.423503636)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4f9f5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c4e3ec80>, 2339910.539456514)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4ec7950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5cca6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c57b6dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c529bc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4fbdb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81172a190>, 2339889.779081875)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5cc96d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810d77af0>, 2339897.67587393)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c57b74d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c67c8360>, 2339904.463590888)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c529bb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c7f74ec0>, 2339909.882884576)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4fbdc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4b01110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c4d0faf0>, 2339927.469909294)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4b01250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4b378d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c7f74130>, 2339917.900498696)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4b34390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4ceb2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3d45d30>, 2339918.002026317)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4c2c990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4ee3490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c4f2d160>, 2339916.143010319)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c51cac50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4ca1650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4cb2210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4cc57d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4cdf390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4ce9410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4b24f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4cb8b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4c73c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4c73bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4d45d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4da5390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4e216d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4e90990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4feb390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4c0fb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c566bf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5309f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5f32450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3261080>, 2339917.973596772)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4ca1490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c7e80280>, 2339917.787965767)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4feacd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ea6c10>, 2339912.710341673)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4cc5850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2a37f50>, 2339912.718686748)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4cdf910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c683b690>, 2339912.769080251)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4ce9f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c4d0f9a0>, 2339917.823444013)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4b414d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0348d70>, 2339912.85314884)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4b35c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c7f75a20>, 2339917.474977277)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4d8a610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81193aa50>, 2339917.827550277)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4d31490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8106759b0>, 2339917.537743101)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4d45010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c5050c90>, 2339917.742837165)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4da6450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8111ef690>, 2339917.326295802)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4e227d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8111ed780>, 2339918.04006009)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4e92050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810ec4520>, 2339914.844420932)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4feaf90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810ec5a90>, 2339937.478774975)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4c0fd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811314c20>, 2339915.838384278)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5669190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c88bc01ef0>, 2339915.844065482)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5658290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810ec4360>, 2339915.896916901)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5f316d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4c2d4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4cdf450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c56dadd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c5051ef0>, 2339912.735740756)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4c2db90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81249c280>, 2339937.518935991)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c54ef0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3d47b60>, 2339915.873871948)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c56b8a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5ebc110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81025e3c0>, 2339937.176619265)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5ebc210>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5faddd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3f4b230>, 2339927.460835383)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5fae690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5d98690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5d0cbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4b026d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5f8e450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3f4a6d0>, 2339918.07989762)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5ddd050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81025d6a0>, 2339925.015184672)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5d0cf90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81025cec0>, 2339925.031908594)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5852e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a154e0>, 2339938.284818538)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5f8d410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4caae10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5d05a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811de52d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81025ec80>, 2339924.983744237)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5fe8d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81025d160>, 2339924.987851242)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5d05750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a17930>, 2339937.289596514)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8119ae710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81057a150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a14fa0>, 2339938.300965504)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5eea090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811d10b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810831e80>, 2339927.210512327)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d17802d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811a454d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811b23d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d380b9a0>, 2339942.328382662)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811a46890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d380ac10>, 2339937.139393682)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c812461b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811374bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5852b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5c88890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c812456850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c88bfa9c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5f0aa50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0d2d050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810265d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110f81a0>, 2339927.064432646)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c88bbdba90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3f4a120>, 2339937.160265089)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811377950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ae3cb0>, 2339927.878435929)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4ebad10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8124cc440>, 2339938.743229262)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5c89410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a16c10>, 2339939.472262033)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c812454690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110fa4a0>, 2339927.1016098)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4c04d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110f9a90>, 2339927.166029053)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0d2f210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e13540>, 2339927.186756487)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8102667d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c5f1d710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110f9010>, 2339937.18883395)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c5e3b750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0ed2f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c6715cc0>, 2339942.188540903)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0ed3250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0b00b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c6717620>, 2339938.158354773)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8108b7250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d116e910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0e3e050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0d594d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0afbad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0cf5850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0b75910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ef1780>, 2339938.205871154)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811928e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ef39a0>, 2339940.688947422)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0e3cd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c6716ac0>, 2339940.830343229)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0d587d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c6714750>, 2339942.08728476)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811274fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c6715240>, 2339938.355235248)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0c226d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c6714360>, 2339940.771904902)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1785ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0bca290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ef1160>, 2339940.506635529)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0bca650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d114c450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c60f80d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ef0c20>, 2339940.388501117)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c88bfddcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c6ae3700>, 2339939.786397181)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c88bbda290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0a19890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c6ae1cc0>, 2339939.470127064)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0d70290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0d96490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a155c0>, 2339940.476490118)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0d94390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0986b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d08c2490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0a78950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0d69b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0c4e2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0918650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110ec520>, 2339940.590347428)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0985d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d09b0090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c6ae2a50>, 2339940.658629629)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0800f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c6ae1e10>, 2339941.435195823)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0a7bb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d380bee0>, 2339942.181995449)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0d68990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d314c280>, 2339941.390899232)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0c4dcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110edd30>, 2339942.074659655)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d091b890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110ef930>, 2339943.167474283)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0a12c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d088a2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810557af0>, 2339944.624375743)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d088bd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d080d110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8105564a0>, 2339942.09967352)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0df80d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d071f810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d076e650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3809b70>, 2339942.907748629)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d071e6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810557380>, 2339942.973288743)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d076c850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d08edcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810554750>, 2339943.174051783)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d08ec150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d055d710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d043d790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110ed9b0>, 2339945.379409545)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d055c6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810554d00>, 2339942.968665053)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d043f8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d04d6690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d05faa90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d087a090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c6ae3e70>, 2339942.695649134)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0e11610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d06313d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d07456d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d390cf30>, 2339944.96111572)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d04d5990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3a14210>, 2339942.055845355)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d05f98d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810556740>, 2339944.946333199)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0633c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d390e970>, 2339944.18994845)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0747ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d067ef10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d390e510>, 2339944.312692583)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d067f510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d04a4b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810747310>, 2339944.974132286)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d04a4090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d05c1190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d04c4e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d390c360>, 2339944.301099825)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0d06a10>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d380b540>, 2339944.631242418)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c60d7410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d088aa50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810554280>, 2339944.658190716)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d088aed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0245390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811c95850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1daa270>, 2339945.156446565)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0c4d7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1da9c50>, 2339945.105321224)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0c79d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0100c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1da8f30>, 2339945.460325004)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0101210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7f89290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1da9780>, 2339946.297240535)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c9b0706490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d03175d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d03dcf50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d03f8d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d047a190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7d36050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0595e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d390d780>, 2339945.126398853)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0314250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110ef690>, 2339946.263803108)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d03de110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110ecbb0>, 2339946.879920869)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d04fb790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1dab460>, 2339946.31773913)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0155850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1da8670>, 2339946.74764052)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7d35b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1da8bb0>, 2339944.000797584)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0595910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d04eafd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110ef380>, 2339946.940738593)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d04e9cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0a3c350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81086fa90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d380a270>, 2339947.087200018)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d05c18d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81182f310>, 2339946.783995097)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c811f0f7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8119a3150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81185a990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810557460>, 2339948.109583445)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8119a1e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1b95b00>, 2339946.76776329)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8118595d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811c8ca90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1da99b0>, 2339946.97177092)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8110dac10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810248550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810dba5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1b97e00>, 2339947.528897803)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0b8d750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1b975b0>, 2339946.936054197)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810dbb310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8118ed450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1daa660>, 2339948.94451253)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d03944d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c81081cb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c4e3e200>, 2339948.227132373)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81081c550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0162010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c8116467d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810559150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c88c0e5910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2ee2650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810556190>, 2339946.356027314)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0162590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2ef3690>, 2339946.456664618)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8116458d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1b95780>, 2339948.219799328)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d04703d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1b95da0>, 2339948.553417126)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c88c0e78d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1b95010>, 2339948.189293356)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81082b090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3ee9fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0162450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8110eeeb0>, 2339948.600556498)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d01cc4d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c4e3c600>, 2339948.808965232)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0162750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810486bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3e7c310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c4e3ce50>, 2339949.258994776)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3a21550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c4e3c130>, 2339948.566963973)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3e7cd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810f58a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810555a90>, 2339948.781070249)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3c40c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3cab450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d396b7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3b00810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ca2660>, 2339949.242638564)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d396a990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d283c750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1daa890>, 2339948.697484536)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2df81d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3926010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d06d81d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d390f8c0>, 2339949.454471574)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c8104bbc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ca00c0>, 2339949.654974172)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3755b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3aea0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1dab850>, 2339949.164047416)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3ae8850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810dd0610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1b971c0>, 2339950.119236927)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d368fe90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2e4be50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c6838360>, 2339950.655410526)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810d6a6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d32b6fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81182f850>, 2339948.79343495)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3cabb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c810f5a210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81162b620>, 2339949.001102969)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d36726d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ca12b0>, 2339950.012863835)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c810f5af50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ca18d0>, 2339950.719490318)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d35e0ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d32d0c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d387e250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c683b380>, 2339951.07156602)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d187e550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c68382f0>, 2339951.051710645)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d387ecd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d19a0550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d174f010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c4e3dfd0>, 2339951.72552354)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1fbdc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c4e3f380>, 2339951.293672197)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d174c7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1e78990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c68383d0>, 2339950.936691493)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1e7ab50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3900f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ca1470>, 2339950.93223552)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3902bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1c943d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8103495c0>, 2339951.746869788)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1c95190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1c18d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1e4f290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81034aa50>, 2339951.084264182)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1c19810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81034b620>, 2339951.379914838)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1b65750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d32e94d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1ae3e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ca2d60>, 2339951.619079114)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d01cf790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1b94b40>, 2339951.245507524)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1ae0ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1aad950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1ca2200>, 2339951.225487786)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d373b810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1cad3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81034b7e0>, 2339951.121113827)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1774450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d20a4ed0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1c21410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810349f60>, 2339951.764832228)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d19d0790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8103490f0>, 2339951.809704673)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1c229d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1dd7ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81034a740>, 2339951.789719834)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1dd6450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2f234d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1a78c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c683bf50>, 2339952.328650702)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2f23810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810459b70>, 2339952.51805228)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1a786d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d176e790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810459ef0>, 2339952.766026054)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c88b164150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1982d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81045b540>, 2339954.698220844)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1981810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d21981d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81045b150>, 2339952.986992955)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d20d87d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d25aa850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81045b930>, 2339952.9619432)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c81047b310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1a4cb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d32b4e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81045aac0>, 2339953.264173605)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1e9f9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c7e80980>, 2339952.853857133)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d21a8150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1ca7710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c7e82890>, 2339953.04371102)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1f7c290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d17671d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d204c2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c6ae3460>, 2339953.281786159)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1766090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c4e3f9a0>, 2339952.967708048)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d204c290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d26fb710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c7e81e80>, 2339954.792492875)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d26f9610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1fcb010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d24f9310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c7e81400>, 2339953.487177763)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d231e390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81045baf0>, 2339954.057098727)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d24f93d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d372b890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810348980>, 2339953.565106806)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1c21010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2409e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c7e81470>, 2339954.692952302)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d240bdd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2609990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8105573f0>, 2339954.769316007)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d173ff90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d196f810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d031f3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81034a430>, 2339954.450652907)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c88b167750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81034b850>, 2339954.787959083)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d031ef90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1e78390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3e4b1c0>, 2339954.758738289)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1f33950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7ef9450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2274290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3e48830>, 2339955.240431512)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1d0c690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0143b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d27830d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3e49da0>, 2339955.470389298)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0143910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c812414750>, 2339955.17543607)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2781b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0618750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8124163c0>, 2339955.383370314)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d0618410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0256b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c6839f60>, 2339956.057472788)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d02569d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d26bf990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c67b4550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d07d2bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d11de250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3e4b850>, 2339955.349446336)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2711450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c7e83930>, 2339953.474699826)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d11dc3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c812417000>, 2339956.730132769)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d26bca50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8124141a0>, 2339955.062827137)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c67b7350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c7e809f0>, 2339953.305317833)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d07d2ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2ca7250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d390f540>, 2339954.735799981)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d24d3f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c68dea90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d188f070>, 2339955.963250144)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c68de790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c68c7b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c67cc4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d188d780>, 2339955.711174714)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c68c6b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c7e814e0>, 2339956.225618905)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c67cc690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6ae4d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d188f380>, 2339956.172843638)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6ae6c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c68a4dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c67c1350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d188c280>, 2339957.135998753)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c67c1510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6753790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c812416270>, 2339957.106470127)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c67539d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c68aa250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d237f9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c81034ae40>, 2339956.707770295)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c88b15d790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c7e81390>, 2339956.518265886)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d237f350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6732750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c664ad50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3501470>, 2339957.507871787)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6731cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3501550>, 2339956.626163207)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1a4d850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d211e910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c672ab90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1b979a0>, 2339958.297431369)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c67a9b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3501e10>, 2339956.941662712)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d27104d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d188c2f0>, 2339955.367156382)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7f32210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c68fa190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c667e6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c6838750>, 2339956.910361393)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c680ee90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c812415f60>, 2339957.413410818)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c667e450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6822c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6835890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c69c5650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d35039a0>, 2339957.584494326)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6836610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3502d60>, 2339958.909455154)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c69c44d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c69994d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3e4a660>, 2339957.565952179)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6999810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6988d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1a85160>, 2339957.755103713)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6989a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6760ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c6839be0>, 2339957.532073558)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6762610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6994450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6779690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d188c600>, 2339958.993397199)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6996990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1a86c10>, 2339958.630173996)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6779ed0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6985c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1a86c80>, 2339958.985828036)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6984d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c68a9890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1a85550>, 2339959.047303072)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6890310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c67c1b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c670dcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d198cd70>, 2339959.013474895)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c67c2490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c696bd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d198cd00>, 2339958.580265077)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6969b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6680e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8124165f0>, 2339957.715233181)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6820d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7c7e81710>, 2339959.129320264)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c670e0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1a86b30>, 2339959.45225283)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c66815d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6752290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d198d240>, 2339959.272592696)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6750c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1d826d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3501da0>, 2339959.431023724)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d1d81690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6900750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d188dd30>, 2339959.965252449)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6902c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6ad8b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d198e4a0>, 2339959.61292583)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6ada250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6afb890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6aa0750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d198de10>, 2339959.893416338)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6af8b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3b2b930>, 2339959.399869648)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6adac90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d18c3490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c8124cd4e0>, 2339960.8338766)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6a64dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6a50090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6bfe750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d188c910>, 2339959.849255885)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6a50690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3b2b1c0>, 2339960.290712962)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6bfe350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c689c4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1a85080>, 2339960.117155617)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c689fc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6bab210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3b2a970>, 2339960.391794206)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6baa110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c694f4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c812414c20>, 2339961.082016418)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c694f550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6b5ac10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c682a890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6912bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d198def0>, 2339959.042263721)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6912dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3502740>, 2339961.037106921)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6b58750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d198fd20>, 2339961.656560617)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6a7d850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6bb6f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3c29d30>, 2339960.824897332)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c67e80d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6b97190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6b1fe10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3b2a580>, 2339961.042879395)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6b972d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3c28130>, 2339961.009968292)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6b1ce50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6cad210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3c2bee0>, 2339961.831416673)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6caef90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6c87090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3c2a5f0>, 2339961.27918385)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6c86590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6ccfa50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6c1c5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d188f5b0>, 2339960.989692234)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6cce150>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811a3d160>, 2339961.189392093)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6c1e550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6c059d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811a3cbb0>, 2339961.140038684)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6c06910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6de58d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811a3cde0>, 2339962.20359325)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6de6310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6db5110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811a3e120>, 2339962.444406677)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6dbc4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6da2250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6d5b0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d188e510>, 2339962.194652395)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6b8a910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811a3c980>, 2339962.310466511)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6d5ac10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6d08550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811a3f5b0>, 2339961.995491782)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6d08050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6d244d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3b2bb60>, 2339962.567391889)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6d25f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c693ae50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d1e5ef50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811a3e0b0>, 2339963.628073266)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6ca4310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3503850>, 2339960.658251386)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6b4e710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6b58710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1a87d20>, 2339963.062628214)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6b58450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6ee4110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811a3f930>, 2339963.001089155)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6ee5950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6ff6d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6d19810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d3ac80>, 2339962.926365634)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6e97790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d198c520>, 2339962.931258001)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6d19110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6e783d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6f92f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d3ab30>, 2339965.289012891)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c68faed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d3b7e0>, 2339963.208528659)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6f90550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6f82fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d390f0>, 2339963.050708421)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6fc1d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6d89590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6f512d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d39e80>, 2339963.802566486)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6b8ac90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d3a120>, 2339963.615388315)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6f50150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6f6d950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d39f60>, 2339963.18025617)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6f6c4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6f4f890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d1a87ee0>, 2339963.692711542)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6f4ff90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6f28b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3c2a660>, 2339964.463761824)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c68b3950>\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6f0f610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3c2acf0>, 2339964.335302784)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6ef23d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6caff50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6e4fe10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3c2b850>, 2339964.424776184)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6e4cf10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0271780>, 2339965.581239033)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c708bd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7052ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0272dd0>, 2339965.205939041)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7050590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c71c7c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d3b3f0>, 2339965.263943601)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c71c7bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6f53c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0273d20>, 2339964.930483744)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c71b66d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c71d9590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c715ee10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0271b00>, 2339964.917076981)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6e1bdd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d30bdd30>, 2339967.002087543)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c715e810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7152fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d30bd9b0>, 2339965.2106259)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7091910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6b5bb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c72e35d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d30bd8d0>, 2339966.837170977)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c71b7290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d3a580>, 2339965.190008897)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c72e19d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6e1ac90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d38600>, 2339965.311492196)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6e19590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7290790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0180c90>, 2339965.832466788)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7290910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c720fb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0180a60>, 2339967.550092724)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c720d7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811a3cf30>, 2339966.54395043)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c71fad50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c70e4e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c71fa310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c72a4d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3c28d00>, 2339965.833724715)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c71f88d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811a3d240>, 2339965.122274567)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c70e4290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d01838c0>, 2339966.88809325)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c72a5b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6bfde10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d198eeb0>, 2339966.598061546)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7333810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7300c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3600280>, 2339967.984109681)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7301cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6d60190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d36030e0>, 2339967.389954757)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7392910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d267e210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3600830>, 2339967.293989858)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d267e810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7344190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c73ee510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3602890>, 2339967.55737792)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c702f010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3600bb0>, 2339967.596601322)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c73ec4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c74e5fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7479990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d36017f0>, 2339967.240738049)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6aa1390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0182dd0>, 2339967.354306324)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7478490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7486bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7432490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d36034d0>, 2339967.288867242)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c73f0310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3603a80>, 2339967.610332689)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7433a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6f824d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0272ac0>, 2339967.637180939)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c74e5c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c700d890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c75b5890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d35e51d0>, 2339969.119327663)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6ecd5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d35e4de0>, 2339968.941634694)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c75b7dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c757bd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c734ca10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c689ee10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0181be0>, 2339966.518210436)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c689f150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811a3d2b0>, 2339968.823440549)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7579150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0272c10>, 2339966.438583249)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c6ff5b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2eb2990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d188e9e0>, 2339970.597315683)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c72787d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c73707d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c75203d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2e00670>, 2339969.113916638)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7370850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2e009f0>, 2339969.038042437)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c75224d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c747b390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2e01b00>, 2339969.255875433)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c759d850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c75816d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2e00c90>, 2339968.77517938)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7583290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c76a1e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3b29cc0>, 2339969.176366762)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c76a2510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c74a2e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c88b150050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811a3e5f0>, 2339969.297611837)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c73eec50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c76da250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2e02740>, 2339969.043283315)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c76d8990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2cb0a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2c98c90>, 2339969.859231959)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c77fd850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c77d7690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d01817f0>, 2339969.29867534)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c77d4e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d3601d30>, 2339971.66343038)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c88b1501d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2572cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c77ad550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2e09bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2c9a200>, 2339970.425339188)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2570c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0270d70>, 2339970.562184945)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c77ac410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2c9af90>, 2339971.135883039)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2e098d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c6f334d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2c9cad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2c9b460>, 2339971.490096463)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c7148b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2c989f0>, 2339971.55135332)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2869390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d0571090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2bec950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811d3af20>, 2339971.458758697)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2976a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2a19bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e51160>, 2339971.808719508)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2a1b0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c764c250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e53e70>, 2339976.111634752)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c764cf10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2f144d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2dbf690>, 2339972.756510816)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2cd9690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2fa1a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2dbeac0>, 2339971.186184596)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2fa1cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3513010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2dbd240>, 2339971.787855345)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d3512c50>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2c65750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2dbec80>, 2339972.25492904)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2c64790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c759d390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d35e43d0>, 2339971.560936362)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2bf5450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d29cb7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2a23d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d2ad9810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7629950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2c9a5f0>, 2339970.370108461)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c73909d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811a3f1c0>, 2339972.218608929)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d29c82d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d30bf4d0>, 2339972.180150196)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2a23510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2dbf5b0>, 2339972.892059994)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2ad9310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2dbcb40>, 2339987.189748883)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2da5890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c811505ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2f32ac0>, 2339987.088758418)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d08ac690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d24edc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2c98590>, 2339976.033591362)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d24ed410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4b0ca90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2f32a50>, 2339973.226111195)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c45cd690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d3105010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2f33c40>, 2339973.300925223)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2a23e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c45fb990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d30bedd0>, 2339987.125447452)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c45fbed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4b595d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4426290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2a342f0>, 2339987.763869511)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4b5afd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2a35d30>, 2339976.366180185)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4426090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4432190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2a36900>, 2339987.732795655)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c44328d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c443d590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2a34b40>, 2339987.335688668)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c443dbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c443f810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4468a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2a34210>, 2339987.177753985)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c443f210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2e01cc0>, 2339987.479049308)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c44688d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4b31c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c449d6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4cefa10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4b2a750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2f33770>, 2339972.881086699)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2a0a6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2f31c50>, 2339987.202316214)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4b31c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2a35e10>, 2339976.396618723)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c449dd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d35e4750>, 2339972.059039715)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4b29550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7d07f2b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e53770>, 2339987.751835209)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d07f2bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c44c9a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e51a20>, 2339997.907071203)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c44c9f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c43d1150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d29d0de0>, 2339978.430442631)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c43d1ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c447a6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c7644090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c44e7050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c44be790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4329550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c43b4b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0183770>, 2339987.345659649)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c447b650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d0270a60>, 2339987.256731521)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7d2da6f90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2dbc4b0>, 2339987.557732887)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4427090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2a356a0>, 2339987.761693359)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c44bef10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2b642f0>, 2339976.506237247)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c431df90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d29d18d0>, 2339998.176212693)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c43b4e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c43fb650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2a34050>, 2339997.791222753)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c43fbf10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4488610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2b65860>, 2339999.747245531)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4488110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c430f010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d29d39a0>, 2339997.649229849)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c430f490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c423c210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2747c40>, 2340000.662429278)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4483c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c427f6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810142b30>, 2339990.33989648)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c427f550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c411d450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2c982f0>, 2340000.771459434)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c411dc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2f31860>, 2340001.203009329)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4140a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4255e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4269750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c42a8c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c42c1e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c42e16d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4104a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d27447c0>, 2340001.401622984)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c424bd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d29d3d90>, 2339999.963138212)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4269e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810143700>, 2339998.334055413)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c42a96d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810143770>, 2340001.128697259)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c42c2810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c811e52c10>, 2340001.702917443)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c42e1550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c810142900>, 2340000.602827093)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c77ed050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c41fe090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d25d1d30>, 2339999.769146729)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c41fdf50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4083a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c414af50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c417b390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4036390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4055510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d288cfa0>, 2340000.037793953)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c407b190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d27ef380>, 2340002.497716771)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c414ad50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d2f31da0>, 2340001.354578937)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c417b910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d25d2660>, 2340001.146365677)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4036b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d25d11d0>, 2340002.492171884)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4432e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c4197090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c41c5750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x14c7c41ccd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d27ed6a0>, 2340003.019308521)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c4197810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d27ee890>, 2340002.756828127)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c41c5f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x14c7d27eeba0>, 2340003.006592882)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x14c7c41cd5d0>\n",
      "Evaluating workflow:  86%|████████▌ | 2572/3000 [08:29<01:03,  6.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2577/3000 [08:30<00:39, 10.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2580/3000 [08:30<00:32, 12.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2583/3000 [08:30<00:31, 13.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2585/3000 [08:30<00:35, 11.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▋ | 2589/3000 [08:32<01:09,  5.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▋ | 2591/3000 [08:32<01:02,  6.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2596/3000 [08:32<00:38, 10.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2598/3000 [08:32<00:48,  8.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2600/3000 [08:33<00:49,  8.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2604/3000 [08:33<00:42,  9.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2606/3000 [08:33<00:52,  7.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2609/3000 [08:34<00:43,  9.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2611/3000 [08:34<00:58,  6.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2613/3000 [08:35<01:04,  6.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2614/3000 [08:35<00:58,  6.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2618/3000 [08:35<00:48,  7.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2622/3000 [08:35<00:34, 10.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2625/3000 [08:35<00:29, 12.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2628/3000 [08:36<00:56,  6.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2630/3000 [08:37<01:02,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2026-01-13 12:51:15.970\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert analyst of perturbation datasets. For K562 cells subjected to perturbation of DHDDS, does the expression profile of ATF7IP2 indicate a significant change relative to control conditions? Reply strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are an expert analyst of perturbation datasets. For K562 cells subjected to perturbation of DHDDS, does the expression profile of ATF7IP2 indicate a significant change relative to control conditions? Reply strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 4527}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 9 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2634/3000 [08:42<03:46,  1.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:51:16.313\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 9 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2635/3000 [08:42<03:32,  1.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:51:25.111\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a genomics expert evaluating perturbation experiments. In K562 cells, CAD is perturbed and the expression of CTD-2587M2.1 is measured. Determine whether CTD-2587M2.1 shows a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are a genomics expert evaluating perturbation experiments. In K562 cells, CAD is perturbed and the expression of CTD-2587M2.1 is measured. Determine whether CTD-2587M2.1 shows a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 1916}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 3 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2636/3000 [08:51<11:27,  1.89s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:51:25.205\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in gene expression analysis. For K562 cells, assess whether perturbation of TIMM44 is associated with a significant change in C17orf64 expression compared with unperturbed controls. Answer strictly as 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are an expert in gene expression analysis. For K562 cells, assess whether perturbation of TIMM44 is associated with a significant change in C17orf64 expression compared with unperturbed controls. Answer strictly as 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 3715}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 3 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:51:25.221\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to SEC61B and then measure expression of PIK3IP1. Does this perturbation cause a significant change in PIK3IP1 expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to SEC61B and then measure expression of PIK3IP1. Does this perturbation cause a significant change in PIK3IP1 expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 2864}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 3 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2638/3000 [08:51<07:56,  1.32s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:51:25.424\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert analyst of perturbation datasets. For K562 cells subjected to perturbation of SRP72, does the expression profile of NOX5 indicate a significant change relative to control conditions? Reply strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are an expert analyst of perturbation datasets. For K562 cells subjected to perturbation of SRP72, does the expression profile of NOX5 indicate a significant change relative to control conditions? Reply strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 3417}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2639/3000 [08:51<06:41,  1.11s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:51:25.468\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in single-cell biology and functional genomics. In K562 cells, SAMM50 is perturbed and the expression of NOXA1 is measured. Does this perturbation cause a significant change in NOXA1 expression? Answer strictly in the format 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are an expert in single-cell biology and functional genomics. In K562 cells, SAMM50 is perturbed and the expression of NOXA1 is measured. Does this perturbation cause a significant change in NOXA1 expression? Answer strictly in the format 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 1410}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2026-01-13 12:51:25.486\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in single-cell biology and functional genomics. In K562 cells, TTI2 is perturbed and the expression of PGM3 is measured. Does this perturbation cause a significant change in PGM3 expression? Answer strictly in the format 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are an expert in single-cell biology and functional genomics. In K562 cells, TTI2 is perturbed and the expression of PGM3 is measured. Does this perturbation cause a significant change in PGM3 expression? Answer strictly in the format 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 2080}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:51:25.504\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a genomics expert evaluating perturbation experiments. In K562 cells, GNPNAT1 is perturbed and the expression of ABI2 is measured. Determine whether ABI2 shows a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are a genomics expert evaluating perturbation experiments. In K562 cells, GNPNAT1 is perturbed and the expression of ABI2 is measured. Determine whether ABI2 shows a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 2026}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:51:25.532\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a domain expert in functional genomics. For experiments carried out in K562 cells, we perturb HSPA5 and examine the expression of HLA-E. Does perturbing HSPA5 lead to a significant change in HLA-E expression? Reply only with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are a domain expert in functional genomics. For experiments carried out in K562 cells, we perturb HSPA5 and examine the expression of HLA-E. Does perturbing HSPA5 lead to a significant change in HLA-E expression? Reply only with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 3261}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2644/3000 [08:51<02:56,  2.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:51:25.613\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a genomics expert evaluating perturbation experiments. In K562 cells, EIF2B4 is perturbed and the expression of RP11-90P5.2 is measured. Determine whether RP11-90P5.2 shows a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are a genomics expert evaluating perturbation experiments. In K562 cells, EIF2B4 is perturbed and the expression of RP11-90P5.2 is measured. Determine whether RP11-90P5.2 shows a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 406}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:51:25.643\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in single-cell biology and functional genomics. In K562 cells, MARS is perturbed and the expression of RUNX1 is measured. Does this perturbation cause a significant change in RUNX1 expression? Answer strictly in the format 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are an expert in single-cell biology and functional genomics. In K562 cells, MARS is perturbed and the expression of RUNX1 is measured. Does this perturbation cause a significant change in RUNX1 expression? Answer strictly in the format 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 2730}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2646/3000 [08:51<02:17,  2.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:51:25.684\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in regulatory genomics. Consider data from K562 cells in which DDOST is perturbed and C9orf64 expression is observed. Does this perturbation lead to a significant difference in C9orf64 expression relative to control conditions? Answer strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are an expert in regulatory genomics. Consider data from K562 cells in which DDOST is perturbed and C9orf64 expression is observed. Does this perturbation lead to a significant difference in C9orf64 expression relative to control conditions? Answer strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 3873}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:51:25.770\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in single-cell biology and functional genomics. In K562 cells, PSMD4 is perturbed and the expression of SMIM14 is measured. Does this perturbation cause a significant change in SMIM14 expression? Answer strictly in the format 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are an expert in single-cell biology and functional genomics. In K562 cells, PSMD4 is perturbed and the expression of SMIM14 is measured. Does this perturbation cause a significant change in SMIM14 expression? Answer strictly in the format 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 3300}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2648/3000 [08:51<01:46,  3.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:51:25.828\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to TTI1 and then measure expression of RP11-16E12.1. Does this perturbation cause a significant change in RP11-16E12.1 expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to TTI1 and then measure expression of RP11-16E12.1. Does this perturbation cause a significant change in RP11-16E12.1 expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 3234}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:51:27.656\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in regulatory genomics. Consider data from K562 cells in which PPWD1 is perturbed and PPP1R16B expression is observed. Does this perturbation lead to a significant difference in PPP1R16B expression relative to control conditions? Answer strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are an expert in regulatory genomics. Consider data from K562 cells in which PPWD1 is perturbed and PPP1R16B expression is observed. Does this perturbation lead to a significant difference in PPP1R16B expression relative to control conditions? Answer strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 4483}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 8 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2650/3000 [08:53<02:46,  2.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:51:28.108\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 7 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:51:28.137\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in gene expression analysis. For K562 cells, assess whether perturbation of YIPF5 is associated with a significant change in CTD-2001C12.1 expression compared with unperturbed controls. Answer strictly as 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are an expert in gene expression analysis. For K562 cells, assess whether perturbation of YIPF5 is associated with a significant change in CTD-2001C12.1 expression compared with unperturbed controls. Answer strictly as 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 575}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 7 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2652/3000 [08:54<02:22,  2.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:51:28.256\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are assisting with the interpretation of perturbation-based expression data. In K562 cells, AARS is perturbed and IDS expression is quantified. Does this perturbation result in a significant change in IDS expression compared with control cells? Respond exactly as 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are assisting with the interpretation of perturbation-based expression data. In K562 cells, AARS is perturbed and IDS expression is quantified. Does this perturbation result in a significant change in IDS expression compared with control cells? Respond exactly as 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 4272}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 7 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:51:28.349\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a genomics expert evaluating perturbation experiments. In K562 cells, HSD17B12 is perturbed and the expression of S100A13 is measured. Determine whether S100A13 shows a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are a genomics expert evaluating perturbation experiments. In K562 cells, HSD17B12 is perturbed and the expression of S100A13 is measured. Determine whether S100A13 shows a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 2786}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 7 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2654/3000 [08:54<01:52,  3.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:51:34.800\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2655/3000 [09:00<07:48,  1.36s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:51:35.379\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▊ | 2656/3000 [09:01<06:55,  1.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:51:35.629\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▊ | 2657/3000 [09:01<05:42,  1.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:51:35.641\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:51:35.652\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:51:35.672\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:51:35.683\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:51:35.827\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▊ | 2662/3000 [09:01<02:20,  2.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2665/3000 [09:03<02:22,  2.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2666/3000 [09:03<02:25,  2.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2668/3000 [09:04<02:03,  2.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2675/3000 [09:05<00:50,  6.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2026-01-13 12:51:39.082\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to CREB1 and then measure expression of C6orf48. Does this perturbation cause a significant change in C6orf48 expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to CREB1 and then measure expression of C6orf48. Does this perturbation cause a significant change in C6orf48 expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 54}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:51:39.382\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2677/3000 [09:05<00:51,  6.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:51:39.788\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are assisting with the interpretation of perturbation-based expression data. In K562 cells, HSPA5 is perturbed and HLA-E expression is quantified. Does this perturbation result in a significant change in HLA-E expression compared with control cells? Respond exactly as 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are assisting with the interpretation of perturbation-based expression data. In K562 cells, HSPA5 is perturbed and HLA-E expression is quantified. Does this perturbation result in a significant change in HLA-E expression compared with control cells? Respond exactly as 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 3262}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2680/3000 [09:06<01:19,  4.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2684/3000 [09:07<00:53,  5.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 2687/3000 [09:07<00:37,  8.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 2689/3000 [09:07<00:39,  7.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 2691/3000 [09:07<00:37,  8.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|████████▉ | 2695/3000 [09:08<00:41,  7.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|█████████ | 2700/3000 [09:08<00:30,  9.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|█████████ | 2704/3000 [09:09<00:31,  9.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2706/3000 [09:09<00:34,  8.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2708/3000 [09:10<00:42,  6.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|█████████ | 2713/3000 [09:10<00:34,  8.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2715/3000 [09:10<00:31,  9.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2717/3000 [09:11<00:27, 10.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2719/3000 [09:11<00:29,  9.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2721/3000 [09:11<00:35,  7.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2723/3000 [09:12<00:47,  5.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2725/3000 [09:12<00:49,  5.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2730/3000 [09:13<00:34,  7.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2732/3000 [09:13<00:29,  9.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2736/3000 [09:13<00:20, 13.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████▏| 2740/3000 [09:14<00:40,  6.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████▏| 2744/3000 [09:15<00:36,  7.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2747/3000 [09:15<00:27,  9.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2753/3000 [09:15<00:17, 13.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2756/3000 [09:16<00:29,  8.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2760/3000 [09:17<00:37,  6.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2765/3000 [09:17<00:24,  9.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2769/3000 [09:17<00:17, 13.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2773/3000 [09:18<00:26,  8.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▎| 2775/3000 [09:18<00:31,  7.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2777/3000 [09:18<00:26,  8.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2779/3000 [09:19<00:32,  6.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2784/3000 [09:19<00:21, 10.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2786/3000 [09:20<00:26,  8.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2788/3000 [09:20<00:32,  6.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2791/3000 [09:20<00:24,  8.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2795/3000 [09:20<00:18, 11.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2797/3000 [09:21<00:30,  6.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2802/3000 [09:21<00:22,  8.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▎| 2807/3000 [09:22<00:20,  9.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▎| 2809/3000 [09:22<00:19, 10.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▎| 2811/3000 [09:23<00:31,  5.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2814/3000 [09:23<00:28,  6.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2817/3000 [09:23<00:20,  9.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2822/3000 [09:24<00:15, 11.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2826/3000 [09:24<00:13, 12.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2828/3000 [09:25<00:20,  8.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2830/3000 [09:25<00:27,  6.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2835/3000 [09:26<00:20,  7.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▍| 2839/3000 [09:26<00:20,  7.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 2840/3000 [09:26<00:25,  6.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 2841/3000 [09:27<00:29,  5.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▍| 2845/3000 [09:27<00:19,  7.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 2848/3000 [09:27<00:14, 10.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▌| 2853/3000 [09:28<00:10, 13.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2855/3000 [09:28<00:13, 10.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▌| 2861/3000 [09:29<00:20,  6.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2863/3000 [09:29<00:17,  7.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2871/3000 [09:30<00:09, 13.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2873/3000 [09:30<00:15,  8.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2875/3000 [09:31<00:16,  7.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2879/3000 [09:31<00:13,  8.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2884/3000 [09:31<00:11, 10.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2886/3000 [09:32<00:11,  9.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▋| 2889/3000 [09:32<00:11,  9.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▋| 2893/3000 [09:33<00:13,  8.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▋| 2895/3000 [09:33<00:17,  5.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2898/3000 [09:34<00:14,  6.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2900/3000 [09:34<00:13,  7.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2905/3000 [09:34<00:11,  8.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2907/3000 [09:34<00:09,  9.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2910/3000 [09:35<00:10,  8.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2914/3000 [09:35<00:08,  9.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2916/3000 [09:36<00:11,  7.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2919/3000 [09:36<00:12,  6.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2922/3000 [09:36<00:09,  7.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2925/3000 [09:37<00:11,  6.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2928/3000 [09:39<00:20,  3.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:52:14.489\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in single-cell biology and functional genomics. In K562 cells, IDH3A is perturbed and the expression of TOB2 is measured. Does this perturbation cause a significant change in TOB2 expression? Answer strictly in the format 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are an expert in single-cell biology and functional genomics. In K562 cells, IDH3A is perturbed and the expression of TOB2 is measured. Does this perturbation cause a significant change in TOB2 expression? Answer strictly in the format 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 980}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 14 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2929/3000 [09:40<00:32,  2.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:52:14.699\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 11 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2930/3000 [09:40<00:29,  2.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:52:27.547\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a domain expert in functional genomics. For experiments carried out in K562 cells, we perturb AMIGO3 and examine the expression of APOL4. Does perturbing AMIGO3 lead to a significant change in APOL4 expression? Reply only with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are a domain expert in functional genomics. For experiments carried out in K562 cells, we perturb AMIGO3 and examine the expression of APOL4. Does perturbing AMIGO3 lead to a significant change in APOL4 expression? Reply only with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 891}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2931/3000 [09:53<03:15,  2.83s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:52:27.755\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in regulatory genomics. Consider data from K562 cells in which EIF2B3 is perturbed and S100A11 expression is observed. Does this perturbation lead to a significant difference in S100A11 expression relative to control conditions? Answer strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are an expert in regulatory genomics. Consider data from K562 cells in which EIF2B3 is perturbed and S100A11 expression is observed. Does this perturbation lead to a significant difference in S100A11 expression relative to control conditions? Answer strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 4643}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 8 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2932/3000 [09:53<02:33,  2.26s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:52:27.853\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in gene regulation studies. For experiments performed in K562 cells, HYOU1 is perturbed and PLA2G15 expression is measured. Determine whether PLA2G15 exhibits a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are an expert in gene regulation studies. For experiments performed in K562 cells, HYOU1 is perturbed and PLA2G15 expression is measured. Determine whether PLA2G15 exhibits a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 3839}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 8 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2934/3000 [09:54<01:34,  1.43s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2026-01-13 12:52:27.983\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 8 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:52:27.998\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in gene regulation studies. For experiments performed in K562 cells, OST4 is perturbed and ATP11B expression is measured. Determine whether ATP11B exhibits a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are an expert in gene regulation studies. For experiments performed in K562 cells, OST4 is perturbed and ATP11B expression is measured. Determine whether ATP11B exhibits a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 299}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 8 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2937/3000 [09:54<00:50,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:52:28.282\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to DHDDS and then measure expression of RP11-304M2.5. Does this perturbation cause a significant change in RP11-304M2.5 expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to DHDDS and then measure expression of RP11-304M2.5. Does this perturbation cause a significant change in RP11-304M2.5 expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 4534}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 8 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:52:28.320\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a genomics expert evaluating perturbation experiments. In K562 cells, MRGBP is perturbed and the expression of RP11-24F11.2 is measured. Determine whether RP11-24F11.2 shows a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are a genomics expert evaluating perturbation experiments. In K562 cells, MRGBP is perturbed and the expression of RP11-24F11.2 is measured. Determine whether RP11-24F11.2 shows a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 1496}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 7 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2939/3000 [09:54<00:36,  1.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:52:28.357\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in single-cell biology and functional genomics. In K562 cells, MARS is perturbed and the expression of DUSP21 is measured. Does this perturbation cause a significant change in DUSP21 expression? Answer strictly in the format 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are an expert in single-cell biology and functional genomics. In K562 cells, MARS is perturbed and the expression of DUSP21 is measured. Does this perturbation cause a significant change in DUSP21 expression? Answer strictly in the format 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 2740}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 7 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:52:28.368\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a domain expert in functional genomics. For experiments carried out in K562 cells, we perturb GNPNAT1 and examine the expression of KLF3. Does perturbing GNPNAT1 lead to a significant change in KLF3 expression? Reply only with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are a domain expert in functional genomics. For experiments carried out in K562 cells, we perturb GNPNAT1 and examine the expression of KLF3. Does perturbing GNPNAT1 lead to a significant change in KLF3 expression? Reply only with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 1981}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 7 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2026-01-13 12:52:28.404\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a genomics expert evaluating perturbation experiments. In K562 cells, NEDD8 is perturbed and the expression of GPRC5C is measured. Determine whether GPRC5C shows a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are a genomics expert evaluating perturbation experiments. In K562 cells, NEDD8 is perturbed and the expression of GPRC5C is measured. Determine whether GPRC5C shows a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 2226}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 7 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:52:28.404\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are assisting with the interpretation of perturbation-based expression data. In K562 cells, SAMM50 is perturbed and NUF2 expression is quantified. Does this perturbation result in a significant change in NUF2 expression compared with control cells? Respond exactly as 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are assisting with the interpretation of perturbation-based expression data. In K562 cells, SAMM50 is perturbed and NUF2 expression is quantified. Does this perturbation result in a significant change in NUF2 expression compared with control cells? Respond exactly as 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 1392}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 7 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:52:28.445\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a genomics expert evaluating perturbation experiments. In K562 cells, EIF2S1 is perturbed and the expression of RP11-3D4.3 is measured. Determine whether RP11-3D4.3 shows a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are a genomics expert evaluating perturbation experiments. In K562 cells, EIF2S1 is perturbed and the expression of RP11-3D4.3 is measured. Determine whether RP11-3D4.3 shows a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 3646}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 7 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2945/3000 [09:54<00:15,  3.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:52:28.535\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to IDH3A and then measure expression of ZEB2. Does this perturbation cause a significant change in ZEB2 expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to IDH3A and then measure expression of ZEB2. Does this perturbation cause a significant change in ZEB2 expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 964}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 7 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:52:35.680\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to SEC61B and then measure expression of SPAST. Does this perturbation cause a significant change in SPAST expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to SEC61B and then measure expression of SPAST. Does this perturbation cause a significant change in SPAST expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 2844}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2948/3000 [10:01<00:46,  1.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:52:35.694\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:52:35.733\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in single-cell biology and functional genomics. In K562 cells, DDOST is perturbed and the expression of ACRV1 is measured. Does this perturbation cause a significant change in ACRV1 expression? Answer strictly in the format 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are an expert in single-cell biology and functional genomics. In K562 cells, DDOST is perturbed and the expression of ACRV1 is measured. Does this perturbation cause a significant change in ACRV1 expression? Answer strictly in the format 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 3890}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-13 12:52:35.837\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in gene expression analysis. For K562 cells, assess whether perturbation of EIF2B2 is associated with a significant change in RP11-838N2.4 expression compared with unperturbed controls. Answer strictly as 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are an expert in gene expression analysis. For K562 cells, assess whether perturbation of EIF2B2 is associated with a significant change in RP11-838N2.4 expression compared with unperturbed controls. Answer strictly as 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 3035}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2951/3000 [10:01<00:31,  1.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-13 12:52:36.679\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:52:36.679\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2953/3000 [10:02<00:28,  1.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:52:36.707\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to EIF2B4 and then measure expression of GDF11. Does this perturbation cause a significant change in GDF11 expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'No', 'question_new': \"You are a specialist in gene perturbation experiments. In K562 cells, we introduce a perturbation to EIF2B4 and then measure expression of GDF11. Does this perturbation cause a significant change in GDF11 expression? Reply only in the form 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 394}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2955/3000 [10:03<00:26,  1.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:52:37.822\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-13 12:52:37.842\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▊| 2958/3000 [10:04<00:18,  2.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▊| 2959/3000 [10:04<00:18,  2.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▊| 2961/3000 [10:05<00:14,  2.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2964/3000 [10:05<00:07,  4.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2967/3000 [10:06<00:10,  3.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2970/3000 [10:07<00:05,  5.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2975/3000 [10:07<00:02,  8.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2977/3000 [10:07<00:03,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2979/3000 [10:08<00:02,  7.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2982/3000 [10:08<00:02,  7.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2984/3000 [10:08<00:02,  6.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|█████████▉| 2988/3000 [10:09<00:01,  7.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2990/3000 [10:09<00:01,  9.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|█████████▉| 2995/3000 [10:09<00:00, 10.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|██████████| 3000/3000 [10:10<00:00,  4.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "Evaluation metrics:  {'f1': 0.0014074595355383533, 'em': 0.0014074595355383533, 'acc': 0.4817030260380014}\n",
      "\u001b[32m2026-01-13 12:52:44.622\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.workflow.workflow_graph\u001b[0m:\u001b[36msave_module\u001b[0m:\u001b[36m1204\u001b[0m - \u001b[1mSaving SEWWorkFlowGraph to debug/optimized_sew_workflow_updatedataset_adamson.json\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# obtain SEWOptimizer after having more roles\n",
    "optimizer = SEWOptimizer(\n",
    "    graph=sew_graph, \n",
    "    evaluator=evaluator, \n",
    "    llm=llm, \n",
    "    max_steps=20,\n",
    "    eval_rounds=1, \n",
    "    repr_scheme=\"python\", \n",
    "    optimize_mode=\"all\", \n",
    "    order=\"zero-order\",\n",
    "    max_rounds=20,\n",
    ")\n",
    "\n",
    "# with suppress_logger_info():\n",
    "#     metrics = optimizer.evaluate(dataset=humaneval, eval_mode=\"test\")\n",
    "# print(\"Evaluation metrics: \", metrics)\n",
    "\n",
    "# optimize the SEW workflow\n",
    "optimizer.optimize(dataset=benchmark)\n",
    "\n",
    "# evaluate the optimized SEW workflow\n",
    "with suppress_logger_info():\n",
    "    metrics = optimizer.evaluate(dataset=benchmark, eval_mode=\"test\")\n",
    "print(\"Evaluation metrics: \", metrics)\n",
    "\n",
    "# save the optimized SEW workflow\n",
    "optimizer.save(\"debug/optimized_sew_workflow_updatedataset_adamson.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "c5f272e3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'f1': 0.0014074595355383533,\n",
       " 'em': 0.0014074595355383533,\n",
       " 'acc': 0.4817030260380014}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ad9c9d1e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5a101dbe",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}